[rubygems/rubygems] Handle base64 encoded checksums in lockfile for f…

…uture compatibility. Save checksums using = as separator. rubygems/rubygems@a36ad7d160
ruby · Oct 23, 2023 · 6dcd4e9 · 6dcd4e9
1 parent c667de7
commit 6dcd4e9
Show file tree

Hide file tree

Showing 6 changed files with 73 additions and 31 deletions.
diff --git a/lib/bundler/checksum.rb b/lib/bundler/checksum.rb
@@ -2,6 2,7 @@
 
 module Bundler
  class Checksum
  ALGO_SEPARATOR = "="
  DEFAULT_ALGORITHM = "sha256"
  private_constant :DEFAULT_ALGORITHM
  DEFAULT_BLOCK_SIZE = 16_384
@@ -15,20 16,24 @@ def from_gem(io, pathname, algo = DEFAULT_ALGORITHM)
  Checksum.new(algo, digest.hexdigest!, Source.new(:gem, pathname))
  end
 
- def from_api(digest, source_uri)
- # transform the bytes from base64 to hex, switch to unpack1 when we drop older rubies
- hexdigest = digest.length == 44 ? digest.unpack("m0").first.unpack("H*").first : digest
-
- if hexdigest.length != 64
- raise ArgumentError, "#{digest.inspect} is not a valid SHA256 hexdigest nor base64digest"
- end
-
- Checksum.new(DEFAULT_ALGORITHM, hexdigest, Source.new(:api, source_uri))
  def from_api(digest, source_uri, algo = DEFAULT_ALGORITHM)
  Checksum.new(algo, to_hexdigest(digest, algo), Source.new(:api, source_uri))
  end
 
  def from_lock(lock_checksum, lockfile_location)
- algo, digest = lock_checksum.strip.split("-", 2)
- Checksum.new(algo, digest, Source.new(:lock, lockfile_location))
  algo, digest = lock_checksum.strip.split(ALGO_SEPARATOR, 2)
  Checksum.new(algo, to_hexdigest(digest, algo), Source.new(:lock, lockfile_location))
  end
 
  def to_hexdigest(digest, algo = DEFAULT_ALGORITHM)
  return digest unless algo == DEFAULT_ALGORITHM
  return digest if digest.match?(/\A[0-9a-f]{64}\z/i)
  if digest.match?(%r{\A[-0-9a-z_ /]{43}={0,2}\z}i)
  digest = digest.tr("-_", " /") # fix urlsafe base64
  # transform to hex. Use unpack1 when we drop older rubies
  return digest.unpack("m0").first.unpack("H*").first
  end
  raise ArgumentError, "#{digest.inspect} is not a valid SHA256 hex or base64 digest"
  end
  end
 
@@ -59,7 64,7 @@ def to_s
  end
 
  def to_lock
- "#{algo}-#{digest}"
  "#{algo}#{ALGO_SEPARATOR}#{digest}"
  end
 
  def merge!(other)
@@ -87,7 92,7 @@ def removal_instructions
  end
 
  def inspect
- abbr = "#{algo}-#{digest[0, 8]}"
  abbr = "#{algo}#{ALGO_SEPARATOR}#{digest[0, 8]}"
  from = "from #{sources.join(" and ")}"
  "#<#{self.class}:#{object_id} #{abbr} #{from}>"
  end
@@ -109,7 114,7 @@ def ==(other)
  end
 
  # phrased so that the usual string format is grammatically correct
- # rake (10.3.2) sha256-abc123 from #{to_s}
  # rake (10.3.2) sha256=abc123 from #{to_s}
  def to_s
  case type
  when :lock

diff --git a/spec/bundler/bundler/lockfile_parser_spec.rb b/spec/bundler/bundler/lockfile_parser_spec.rb
@@ -23,7 23,7 @@
  rake
 
  CHECKSUMS
- rake (10.3.2) sha256-814828c34f1315d7e7b7e8295184577cc4e969bad6156ac069d02d63f58d82e8
  rake (10.3.2) sha256=814828c34f1315d7e7b7e8295184577cc4e969bad6156ac069d02d63f58d82e8
 
  RUBY VERSION
  ruby 2.1.3p242
@@ -121,8 121,8 @@
  let(:lockfile_path) { Bundler.default_lockfile.relative_path_from(Dir.pwd) }
  let(:rake_checksum) do
  Bundler::Checksum.from_lock(
- "sha256-814828c34f1315d7e7b7e8295184577cc4e969bad6156ac069d02d63f58d82e8",
- "#{lockfile_path}:??:1"
  "sha256=814828c34f1315d7e7b7e8295184577cc4e969bad6156ac069d02d63f58d82e8",
  "#{lockfile_path}:20:17"
  )
  end
 
@@ -163,8 163,33 @@
  include_examples "parsing"
  end
 
  context "when the checksum is urlsafe base64 encoded" do
  let(:lockfile_contents) do
  super().sub(
  "sha256=814828c34f1315d7e7b7e8295184577cc4e969bad6156ac069d02d63f58d82e8",
  "sha256=gUgow08TFdfnt-gpUYRXfMTpabrWFWrAadAtY_WNgug="
  )
  end
  include_examples "parsing"
  end
 
  context "when the checksum is of an unknown algorithm" do
  let(:lockfile_contents) do
  super().sub(
  "sha256=",
  "sha512=pVDn9GLmcFkz8vj1ueiVxj5uGKkAyaqYjEX8zG6L5O4BeVg3wANaKbQdpj/B82Nd/MHVszy6polHcyotUdwilQ==,sha256="
  )
  end
  include_examples "parsing"
 
  it "preserves the checksum as is" do
  checksum = subject.sources.last.checksum_store.fetch(specs.last, "sha512")
  expect(checksum.algo).to eq("sha512")
  end
  end
 
  context "when CHECKSUMS has duplicate checksums in the lockfile that don't match" do
- let(:bad_checksum) { "sha256-c0ffee11c0ffee11c0ffee11c0ffee11c0ffee11c0ffee11c0ffee11c0ffee11" }
  let(:bad_checksum) { "sha256=c0ffee11c0ffee11c0ffee11c0ffee11c0ffee11c0ffee11c0ffee11c0ffee11" }
  let(:lockfile_contents) { super().split(/(?<=CHECKSUMS\n)/m).insert(1, " rake (10.3.2) #{bad_checksum}\n").join }
 
  it "raises a security error" do

diff --git a/spec/bundler/commands/lock_spec.rb b/spec/bundler/commands/lock_spec.rb
@@ -214,7 214,7 @@
  end
 
  it "preserves unknown checksum algorithms" do
- lockfile @lockfile.gsub(/(sha256-[a-f0-9] )$/, "constant-true,\\1,xyz-123")
  lockfile @lockfile.gsub(/(sha256=[a-f0-9] )$/, "constant=true,\\1,xyz=123")
 
  previous_lockfile = read_lockfile
 

diff --git a/spec/bundler/install/gemfile/sources_spec.rb b/spec/bundler/install/gemfile/sources_spec.rb
@@ -129,7 129,7 @@
  end
 
  it "works in standalone mode", :bundler => "< 3" do
- gem_checksum = checksum_for_repo_gem(gem_repo4, "foo", "1.0").split("-").last
  gem_checksum = checksum_for_repo_gem(gem_repo4, "foo", "1.0").split(Bundler::Checksum::ALGO_SEPARATOR).last
  bundle "install --standalone", :artifice => "compact_index", :env => { "BUNDLER_SPEC_FOO_CHECKSUM" => gem_checksum }
  end
  end
@@ -337,7 337,7 @@
  expect(err).to eq(<<~E.strip)
  [DEPRECATED] Your Gemfile contains multiple global sources. Using `source` more than once without a block is a security risk, and may result in installing unexpected gems. To resolve this warning, use a block to indicate which gems should come from the secondary source.
  Bundler found mismatched checksums. This is a potential security risk.
- rack (1.0.0) sha256-#{rack_checksum}
  rack (1.0.0) sha256=#{rack_checksum}
  from the API at https://gem.repo2/
  and the API at https://gem.repo1/
  #{checksum_for_repo_gem(gem_repo2, "rack", "1.0.0")}
@@ -354,7 354,7 @@
  end
 
  it "installs from the other source and warns about ambiguous gems when the sources have the same checksum", :bundler => "< 3" do
- gem_checksum = checksum_for_repo_gem(gem_repo2, "rack", "1.0.0").split("-").last
  gem_checksum = checksum_for_repo_gem(gem_repo2, "rack", "1.0.0").split(Bundler::Checksum::ALGO_SEPARATOR).last
  bundle :install, :artifice => "compact_index", :env => { "BUNDLER_SPEC_RACK_CHECKSUM" => gem_checksum, "DEBUG" => "1" }
 
  expect(err).to include("Warning: the gem 'rack' was found in multiple sources.")
@@ -1302,16 1302,16 @@
 
  bundle "install", :artifice => "compact_index", :raise_on_error => false
 
- api_checksum1 = checksum_for_repo_gem(gem_repo1, "rack", "0.9.1").split("sha256-").last
- api_checksum3 = checksum_for_repo_gem(gem_repo3, "rack", "0.9.1").split("sha256-").last
  api_checksum1 = checksum_for_repo_gem(gem_repo1, "rack", "0.9.1").split("sha256=").last
  api_checksum3 = checksum_for_repo_gem(gem_repo3, "rack", "0.9.1").split("sha256=").last
 
  expect(exitstatus).to eq(37)
  expect(err).to eq(<<~E.strip)
  [DEPRECATED] Your lockfile contains a single rubygems source section with multiple remotes, which is insecure. Make sure you run `bundle install` in non frozen mode and commit the result to make your lockfile secure.
  Bundler found mismatched checksums. This is a potential security risk.
- rack (0.9.1) sha256-#{api_checksum3}
  rack (0.9.1) sha256=#{api_checksum3}
  from the API at https://gem.repo3/
- rack (0.9.1) sha256-#{api_checksum1}
  rack (0.9.1) sha256=#{api_checksum1}
  from the API at https://gem.repo1/
 
  Mismatched checksums each have an authoritative source:

diff --git a/spec/bundler/install/gems/compact_index_spec.rb b/spec/bundler/install/gems/compact_index_spec.rb
@@ -876,13 876,25 @@ def start
  end
 
  describe "checksum validation" do
  it "handles checksums from the server in base64" do
  api_checksum = checksum_for_repo_gem(gem_repo1, "rack", "1.0.0").split("sha256=").last
  rack_checksum = [[api_checksum].pack("H*")].pack("m0")
  install_gemfile <<-G, :artifice => "compact_index", :env => { "BUNDLER_SPEC_RACK_CHECKSUM" => rack_checksum }
  source "#{source_uri}"
  gem "rack"
  G
 
  expect(out).to include("Fetching gem metadata from #{source_uri}")
  expect(the_bundle).to include_gems("rack 1.0.0")
  end
 
  it "raises when the checksum does not match" do
  install_gemfile <<-G, :artifice => "compact_index_wrong_gem_checksum", :raise_on_error => false
  source "#{source_uri}"
  gem "rack"
  G
 
- api_checksum = checksum_for_repo_gem(gem_repo1, "rack", "1.0.0").split("sha256-").last
  api_checksum = checksum_for_repo_gem(gem_repo1, "rack", "1.0.0").split("sha256=").last
 
  gem_path = if Bundler.feature_flag.global_gem_cache?
  default_cache_path.dirname.join("cache", "gems", "localgemserver.test.80.dd34752a738ee965a2a4298dc16db6c5", "rack-1.0.0.gem")
@@ -893,9 905,9 @@ def start
  expect(exitstatus).to eq(37)
  expect(err).to eq <<~E.strip
  Bundler found mismatched checksums. This is a potential security risk.
- rack (1.0.0) sha256-2222222222222222222222222222222222222222222222222222222222222222
  rack (1.0.0) sha256=2222222222222222222222222222222222222222222222222222222222222222
  from the API at http://localgemserver.test/
- rack (1.0.0) sha256-#{api_checksum}
  rack (1.0.0) sha256=#{api_checksum}
  from the gem at #{gem_path}
 
  If you trust the API at http://localgemserver.test/, to resolve this issue you can:
@@ -913,7 925,7 @@ def start
  gem "rack"
  G
  expect(exitstatus).to eq(14)
- expect(err).to include("Invalid checksum for rack-0.9.1: \"checksum!\" is not a valid SHA256 hexdigest nor base64digest")
  expect(err).to include('Invalid checksum for rack-0.9.1: "checksum!" is not a valid SHA256 hex or base64 digest')
  end
 
  it "does not raise when disable_checksum_validation is set" do

diff --git a/spec/bundler/support/checksums.rb b/spec/bundler/support/checksums.rb
@@ -61,7 61,7 @@ def remove_checksums_from_lockfile(lockfile, *prefixes)
 
  checksums = checksums.each_line.map do |line|
  if prefixes.nil? || line.match?(prefixes)
- line.gsub(/ sha256-[a-f0-9]{64}/i, "")
  line.gsub(/ sha256=[a-f0-9]{64}/i, "")
  else
  line
  end