From eb92e02be91f7bb64cfc2d745e9ae2212a07d38e Mon Sep 17 00:00:00 2001
From: salma-samy <salmasamy@google.com>
Date: Thu, 19 Oct 2023 03:57:06 -0700
Subject: [PATCH] Add --force option to fetch
This option can be used alongside any fetch operation to compel the fetching process.
PiperOrigin-RevId: 574810048
Change-Id: I3df8fc18ac701cf0ca7a0ce68f1565fb064d59b5
---
.../lib/bazel/commands/FetchCommand.java | 12 ++++-
.../lib/bazel/commands/FetchOptions.java | 12 +++--
.../build/lib/bazel/commands/fetch.txt | 1 +
src/test/py/bazel/bzlmod/bazel_fetch_test.py | 46 +++++++++++++++++++
4 files changed, 67 insertions(+), 4 deletions(-)
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/FetchCommand.java b/src/main/java/com/google/devtools/build/lib/bazel/commands/FetchCommand.java
index e74b9ac6c03..d3a79e668c5 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/commands/FetchCommand.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/FetchCommand.java
@@ -41,6 +41,7 @@ import com.google.devtools.build.lib.query2.engine.QueryException;
import com.google.devtools.build.lib.query2.engine.QueryExpression;
import com.google.devtools.build.lib.query2.engine.QuerySyntaxException;
import com.google.devtools.build.lib.query2.engine.ThreadSafeOutputFormatterCallback;
+import com.google.devtools.build.lib.rules.repository.RepositoryDelegatorFunction;
import com.google.devtools.build.lib.rules.repository.RepositoryDirectoryValue;
import com.google.devtools.build.lib.runtime.BlazeCommand;
import com.google.devtools.build.lib.runtime.BlazeCommandResult;
@@ -52,6 +53,7 @@ import com.google.devtools.build.lib.runtime.commands.QueryCommand;
import com.google.devtools.build.lib.server.FailureDetails;
import com.google.devtools.build.lib.server.FailureDetails.FailureDetail;
import com.google.devtools.build.lib.server.FailureDetails.FetchCommand.Code;
+import com.google.devtools.build.lib.skyframe.PrecomputedValue;
import com.google.devtools.build.lib.skyframe.RepositoryMappingValue.RepositoryMappingResolutionException;
import com.google.devtools.build.lib.skyframe.SkyframeExecutor;
import com.google.devtools.build.lib.util.AbruptExitException;
@@ -82,7 +84,6 @@ import net.starlark.java.eval.EvalException;
allowResidue = true,
completion = "label")
public final class FetchCommand implements BlazeCommand {
- // TODO(kchodorow): add an option to force-fetch targets, even if they're already downloaded.
// TODO(kchodorow): this would be a great time to check for difference and invalidate the upward
// transitive closure for local repositories.
@@ -119,6 +120,15 @@ public final class FetchCommand implements BlazeCommand {
/* showProgress= */ true,
/* id= */ null));
BlazeCommandResult result;
+ if (fetchOptions.force) {
+ // Using commandId as the value -instead of true/false- to make sure to invalidate skyframe
+ // and to actually force fetch each time
+ env.getSkyframeExecutor()
+ .injectExtraPrecomputedValues(
+ ImmutableList.of(
+ PrecomputedValue.injected(
+ RepositoryDelegatorFunction.FORCE_FETCH, env.getCommandId().toString())));
+ }
if (fetchOptions.all || fetchOptions.configure) {
result = fetchAll(env, options, threadsOption, fetchOptions.configure);
} else if (!fetchOptions.repos.isEmpty()) {
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/FetchOptions.java b/src/main/java/com/google/devtools/build/lib/bazel/commands/FetchOptions.java
index c35ebf21878..e5801a1581a 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/commands/FetchOptions.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/FetchOptions.java
@@ -52,7 +52,13 @@ public class FetchOptions extends OptionsBase {
+ " {@@canonical_repo_name}. Only works when --enable_bzlmod is on.")
public List<String> repos;
- /*TODO(salmasamy) add more options:
- * force: to force fetch even if a repo exists
- */
+ @Option(
+ name = "force",
+ defaultValue = "false",
+ documentationCategory = OptionDocumentationCategory.EXECUTION_STRATEGY,
+ effectTags = {OptionEffectTag.CHANGES_INPUTS},
+ help =
+ "Ignore existing repository if any and force fetch the repository again. Only works when "
+ + "--enable_bzlmod is on.")
+ public boolean force;
}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/fetch.txt b/src/main/java/com/google/devtools/build/lib/bazel/commands/fetch.txt
index 8dd87ff0bb5..ce584f7fd5a 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/commands/fetch.txt
+++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/fetch.txt
@@ -5,6 +5,7 @@ Usage: %{product} %{command} [<option> ...]
- --configure: Only fetches repositories marked as 'configure', only works with bzlmod.
- --repo={repo}: Only fetches the specified repository, which can be either {@apparent_repo_name} or {@@canonical_repo_name}
, only works with bzlmod.
+ - --force: Ignore existing repository if any and force fetch the repository again, only works with bzlmod.
- <targets>: Fetches dependencies only for given targets.
%{options}
diff --git a/src/test/py/bazel/bzlmod/bazel_fetch_test.py b/src/test/py/bazel/bzlmod/bazel_fetch_test.py
index a6f1a728837..22fba4125b2 100644
--- a/src/test/py/bazel/bzlmod/bazel_fetch_test.py
+++ b/src/test/py/bazel/bzlmod/bazel_fetch_test.py
@@ -219,6 +219,52 @@ class BazelFetchTest(test_base.TestBase):
stderr,
)
+ def testForceFetch(self):
+ self.ScratchFile(
+ 'MODULE.bazel',
+ [
+ 'ext = use_extension("extension.bzl", "ext")',
+ 'use_repo(ext, "hello")',
+ 'local_path_override(module_name="bazel_tools", path="tools_mock")',
+ 'local_path_override(module_name="local_config_platform", ',
+ 'path="platforms_mock")',
+ ],
+ )
+ self.ScratchFile('BUILD')
+ self.ScratchFile('orange_juice.txt', ['Orange Juice'])
+ file_path = self.Path('orange_juice.txt').replace('\\', '\\\\')
+ self.ScratchFile(
+ 'extension.bzl',
+ [
+ 'def _repo_rule_impl(ctx):',
+ ' file_content = ctx.read("' + file_path + '").strip()',
+ ' print(file_content)',
+ ' ctx.file("WORKSPACE")',
+ ' ctx.file("BUILD")',
+ 'repo_rule = repository_rule(implementation=_repo_rule_impl)',
+ '',
+ 'def _ext_impl(ctx):',
+ ' repo_rule(name="hello")',
+ 'ext = module_extension(implementation=_ext_impl)',
+ ],
+ )
+
+ _, _, stderr = self.RunBazel(['fetch', '--repo=@hello'])
+ self.assertIn('Orange Juice', ''.join(stderr))
+
+ # Change file content and run WITHOUT force, assert no fetching!
+ self.ScratchFile('orange_juice.txt', ['No more Orange Juice!'])
+ _, _, stderr = self.RunBazel(['fetch', '--repo=@hello'])
+ self.assertNotIn('No more Orange Juice!', ''.join(stderr))
+
+ # Run again WITH --force and assert fetching
+ _, _, stderr = self.RunBazel(['fetch', '--repo=@hello', '--force'])
+ self.assertIn('No more Orange Juice!', ''.join(stderr))
+
+ # One more time to validate force is invoked and not cached by skyframe
+ _, _, stderr = self.RunBazel(['fetch', '--repo=@hello', '--force'])
+ self.assertIn('No more Orange Juice!', ''.join(stderr))
+
if __name__ == '__main__':
absltest.main()
--
GitLab