[PATCH v2] submodule: use subrepo algo when fetchig commits

[Michael Schroeder <mls@xxxxxxx>]

If a submodule uses a hash algorithm with shorter hash length than
what is used in the main repository, the recorded submodule commit
is padded with zeros. This is usually not a problem as the default
is to do submodule clones non-shallow and the commit can be found
in the local objects.

But this is not true if the --shallow-submodules clone option is
used (or the --depth option in the submodule update call).
In this case, the commit is often not reachable and a fetch of the
specific commit is done. But the fetch cannot deal with the zero
padding and interprets the commit as a name. Because of this,
the checkout will fail.

Use the subrepo algorithm when converting the stored commit to hex
so that the result matches the expected oid length of the submodule.

Signed-off-by: Michael Schroeder <mls@xxxxxxx>
---
 builtin/submodule--helper.c | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/builtin/submodule--helper.c b/builtin/submodule--helper.c
index 07a1935cbe..5c2e96b517 100644
--- a/builtin/submodule--helper.c
+++ b/builtin/submodule--helper.c
@@ -72,7 +72,7 @@ static char *resolve_relative_url(const char *rel_url, const char *up_path, int
 	return resolved_url;
 }
 
-static int get_default_remote_submodule(const char *module_path, char **default_remote)
+static int get_default_remote_submodule(const char *module_path, char **default_remote, const struct git_hash_algo **hash_algo)
 {
 	const struct submodule *sub;
 	struct repository subrepo;
@@ -106,6 +106,9 @@ static int get_default_remote_submodule(const char *module_path, char **default_
 
 	*default_remote = xstrdup(remote_name);
 
+	if (hash_algo)
+		*hash_algo = subrepo.hash_algo;
+
 	repo_clear(&subrepo);
 	free(url);
 
@@ -1272,7 +1275,7 @@ static void sync_submodule(const char *path, const char *prefix,
 		goto cleanup;
 
 	strbuf_reset(&sb);
-	code = get_default_remote_submodule(path, &default_remote);
+	code = get_default_remote_submodule(path, &default_remote, NULL);
 	if (code)
 		exit(code);
 
@@ -2319,16 +2322,18 @@ static int fetch_in_submodule(const char *module_path, int depth, int quiet,
 	if (depth)
 		strvec_pushf(&cp.args, "--depth=%d", depth);
 	if (oid) {
-		char *hex = oid_to_hex(oid);
+		const struct git_hash_algo *hash_algo = NULL;
+		char *hex;
 		char *remote;
 		int code;
 
-		code = get_default_remote_submodule(module_path, &remote);
+		code = get_default_remote_submodule(module_path, &remote, &hash_algo);
 		if (code) {
 			child_process_clear(&cp);
 			return code;
 		}
 
+		hex = hash_to_hex_algop(oid->hash, hash_algo);
 		strvec_pushl(&cp.args, remote, hex, NULL);
 		free(remote);
 	}
@@ -2635,7 +2640,7 @@ static int update_submodule(struct update_data *update_data)
 		char *remote_ref;
 		int code;
 
-		code = get_default_remote_submodule(update_data->sm_path, &remote_name);
+		code = get_default_remote_submodule(update_data->sm_path, &remote_name, NULL);
 		if (code)
 			return code;
 		code = remote_submodule_branch(update_data->sm_path, &branch);
-- 
2.50.0