Merge branch 'ds/multi-pack-verify'
authorJunio C Hamano <gitster@pobox.com>
Wed, 10 Oct 2018 03:37:16 +0000 (12:37 +0900)
committerJunio C Hamano <gitster@pobox.com>
Wed, 10 Oct 2018 03:37:16 +0000 (12:37 +0900)
"git multi-pack-index" learned to detect corruption in the .midx
file it uses, and this feature has been integrated into "git fsck".

* ds/multi-pack-verify:
  fsck: verify multi-pack-index
  multi-pack-index: report progress during 'verify'
  multi-pack-index: verify object offsets
  multi-pack-index: fix 32-bit vs 64-bit size check
  multi-pack-index: verify oid lookup order
  multi-pack-index: verify oid fanout order
  multi-pack-index: verify missing pack
  multi-pack-index: verify packname order
  multi-pack-index: verify corrupt chunk lookup table
  multi-pack-index: verify bad header
  multi-pack-index: add 'verify' verb

Documentation/git-multi-pack-index.txt
builtin/fsck.c
builtin/multi-pack-index.c
midx.c
midx.h
t/t5319-multi-pack-index.sh

index 1f97e79..f7778a2 100644 (file)
@@ -27,6 +27,10 @@ write::
        When given as the verb, write a new MIDX file to
        `<dir>/packs/multi-pack-index`.
 
+verify::
+       When given as the verb, verify the contents of the MIDX file
+       at `<dir>/packs/multi-pack-index`.
+
 
 EXAMPLES
 --------
@@ -43,6 +47,12 @@ $ git multi-pack-index write
 $ git multi-pack-index --object-dir <alt> write
 -----------------------------------------------
 
+* Verify the MIDX file for the packfiles in the current .git folder.
++
+-----------------------------------------------
+$ git multi-pack-index verify
+-----------------------------------------------
+
 
 SEE ALSO
 --------
index 63c8578..06eb421 100644 (file)
@@ -848,5 +848,23 @@ int cmd_fsck(int argc, const char **argv, const char *prefix)
                }
        }
 
+       if (!git_config_get_bool("core.multipackindex", &i) && i) {
+               struct child_process midx_verify = CHILD_PROCESS_INIT;
+               const char *midx_argv[] = { "multi-pack-index", "verify", NULL, NULL, NULL };
+
+               midx_verify.argv = midx_argv;
+               midx_verify.git_cmd = 1;
+               if (run_command(&midx_verify))
+                       errors_found |= ERROR_COMMIT_GRAPH;
+
+               prepare_alt_odb(the_repository);
+               for (alt =  the_repository->objects->alt_odb_list; alt; alt = alt->next) {
+                       midx_argv[2] = "--object-dir";
+                       midx_argv[3] = alt->path;
+                       if (run_command(&midx_verify))
+                               errors_found |= ERROR_COMMIT_GRAPH;
+               }
+       }
+
        return errors_found;
 }
index 2633efd..fca70f8 100644 (file)
@@ -5,7 +5,7 @@
 #include "midx.h"
 
 static char const * const builtin_multi_pack_index_usage[] = {
-       N_("git multi-pack-index [--object-dir=<dir>] write"),
+       N_("git multi-pack-index [--object-dir=<dir>] (write|verify)"),
        NULL
 };
 
@@ -42,6 +42,8 @@ int cmd_multi_pack_index(int argc, const char **argv,
 
        if (!strcmp(argv[0], "write"))
                return write_midx_file(opts.object_dir);
+       if (!strcmp(argv[0], "verify"))
+               return verify_midx_file(opts.object_dir);
 
        die(_("unrecognized verb: %s"), argv[0]);
 }
diff --git a/midx.c b/midx.c
index f3e8dbc..713d6f9 100644 (file)
--- a/midx.c
+++ b/midx.c
@@ -7,6 +7,7 @@
 #include "object-store.h"
 #include "sha1-lookup.h"
 #include "midx.h"
+#include "progress.h"
 
 #define MIDX_SIGNATURE 0x4d494458 /* "MIDX" */
 #define MIDX_VERSION 1
@@ -76,24 +77,18 @@ struct multi_pack_index *load_multi_pack_index(const char *object_dir, int local
        m->local = local;
 
        m->signature = get_be32(m->data);
-       if (m->signature != MIDX_SIGNATURE) {
-               error(_("multi-pack-index signature 0x%08x does not match signature 0x%08x"),
+       if (m->signature != MIDX_SIGNATURE)
+               die(_("multi-pack-index signature 0x%08x does not match signature 0x%08x"),
                      m->signature, MIDX_SIGNATURE);
-               goto cleanup_fail;
-       }
 
        m->version = m->data[MIDX_BYTE_FILE_VERSION];
-       if (m->version != MIDX_VERSION) {
-               error(_("multi-pack-index version %d not recognized"),
+       if (m->version != MIDX_VERSION)
+               die(_("multi-pack-index version %d not recognized"),
                      m->version);
-               goto cleanup_fail;
-       }
 
        hash_version = m->data[MIDX_BYTE_HASH_VERSION];
-       if (hash_version != MIDX_HASH_VERSION) {
-               error(_("hash version %u does not match"), hash_version);
-               goto cleanup_fail;
-       }
+       if (hash_version != MIDX_HASH_VERSION)
+               die(_("hash version %u does not match"), hash_version);
        m->hash_len = MIDX_HASH_LEN;
 
        m->num_chunks = m->data[MIDX_BYTE_NUM_CHUNKS];
@@ -106,6 +101,9 @@ struct multi_pack_index *load_multi_pack_index(const char *object_dir, int local
                uint64_t chunk_offset = get_be64(m->data + MIDX_HEADER_SIZE + 4 +
                                                 MIDX_CHUNKLOOKUP_WIDTH * i);
 
+               if (chunk_offset >= m->data_len)
+                       die(_("invalid chunk offset (too large)"));
+
                switch (chunk_id) {
                        case MIDX_CHUNKID_PACKNAMES:
                                m->chunk_pack_names = m->data + chunk_offset;
@@ -160,12 +158,10 @@ struct multi_pack_index *load_multi_pack_index(const char *object_dir, int local
 
                cur_pack_name += strlen(cur_pack_name) + 1;
 
-               if (i && strcmp(m->pack_names[i], m->pack_names[i - 1]) <= 0) {
-                       error(_("multi-pack-index pack names out of order: '%s' before '%s'"),
+               if (i && strcmp(m->pack_names[i], m->pack_names[i - 1]) <= 0)
+                       die(_("multi-pack-index pack names out of order: '%s' before '%s'"),
                              m->pack_names[i - 1],
                              m->pack_names[i]);
-                       goto cleanup_fail;
-               }
        }
 
        return m;
@@ -202,7 +198,8 @@ int prepare_midx_pack(struct multi_pack_index *m, uint32_t pack_int_id)
        struct strbuf pack_name = STRBUF_INIT;
 
        if (pack_int_id >= m->num_packs)
-               BUG("bad pack-int-id");
+               die(_("bad pack-int-id: %u (%u total packs"),
+                   pack_int_id, m->num_packs);
 
        if (m->packs[pack_int_id])
                return 0;
@@ -241,7 +238,7 @@ static off_t nth_midxed_offset(struct multi_pack_index *m, uint32_t pos)
        offset32 = get_be32(offset_data + sizeof(uint32_t));
 
        if (m->chunk_large_offsets && offset32 & MIDX_LARGE_OFFSET_NEEDED) {
-               if (sizeof(offset32) < sizeof(uint64_t))
+               if (sizeof(off_t) < sizeof(uint64_t))
                        die(_("multi-pack-index stores a 64-bit offset, but off_t is too small"));
 
                offset32 ^= MIDX_LARGE_OFFSET_NEEDED;
@@ -928,3 +925,83 @@ void clear_midx_file(const char *object_dir)
 
        free(midx);
 }
+
+static int verify_midx_error;
+
+static void midx_report(const char *fmt, ...)
+{
+       va_list ap;
+       verify_midx_error = 1;
+       va_start(ap, fmt);
+       vfprintf(stderr, fmt, ap);
+       fprintf(stderr, "\n");
+       va_end(ap);
+}
+
+int verify_midx_file(const char *object_dir)
+{
+       uint32_t i;
+       struct progress *progress = NULL;
+       struct multi_pack_index *m = load_multi_pack_index(object_dir, 1);
+       verify_midx_error = 0;
+
+       if (!m)
+               return 0;
+
+       for (i = 0; i < m->num_packs; i++) {
+               if (prepare_midx_pack(m, i))
+                       midx_report("failed to load pack in position %d", i);
+       }
+
+       for (i = 0; i < 255; i++) {
+               uint32_t oid_fanout1 = ntohl(m->chunk_oid_fanout[i]);
+               uint32_t oid_fanout2 = ntohl(m->chunk_oid_fanout[i + 1]);
+
+               if (oid_fanout1 > oid_fanout2)
+                       midx_report(_("oid fanout out of order: fanout[%d] = %"PRIx32" > %"PRIx32" = fanout[%d]"),
+                                   i, oid_fanout1, oid_fanout2, i + 1);
+       }
+
+       for (i = 0; i < m->num_objects - 1; i++) {
+               struct object_id oid1, oid2;
+
+               nth_midxed_object_oid(&oid1, m, i);
+               nth_midxed_object_oid(&oid2, m, i + 1);
+
+               if (oidcmp(&oid1, &oid2) >= 0)
+                       midx_report(_("oid lookup out of order: oid[%d] = %s >= %s = oid[%d]"),
+                                   i, oid_to_hex(&oid1), oid_to_hex(&oid2), i + 1);
+       }
+
+       progress = start_progress(_("Verifying object offsets"), m->num_objects);
+       for (i = 0; i < m->num_objects; i++) {
+               struct object_id oid;
+               struct pack_entry e;
+               off_t m_offset, p_offset;
+
+               nth_midxed_object_oid(&oid, m, i);
+               if (!fill_midx_entry(&oid, &e, m)) {
+                       midx_report(_("failed to load pack entry for oid[%d] = %s"),
+                                   i, oid_to_hex(&oid));
+                       continue;
+               }
+
+               if (open_pack_index(e.p)) {
+                       midx_report(_("failed to load pack-index for packfile %s"),
+                                   e.p->pack_name);
+                       break;
+               }
+
+               m_offset = e.offset;
+               p_offset = find_pack_entry_one(oid.hash, e.p);
+
+               if (m_offset != p_offset)
+                       midx_report(_("incorrect object offset for oid[%d] = %s: %"PRIx64" != %"PRIx64),
+                                   i, oid_to_hex(&oid), m_offset, p_offset);
+
+               display_progress(progress, i + 1);
+       }
+       stop_progress(&progress);
+
+       return verify_midx_error;
+}
diff --git a/midx.h b/midx.h
index a210f1a..ce80b91 100644 (file)
--- a/midx.h
+++ b/midx.h
@@ -43,5 +43,6 @@ int prepare_multi_pack_index_one(struct repository *r, const char *object_dir, i
 
 int write_midx_file(const char *object_dir);
 void clear_midx_file(const char *object_dir);
+int verify_midx_file(const char *object_dir);
 
 #endif
index 6f56b38..bd8e841 100755 (executable)
@@ -150,6 +150,125 @@ test_expect_success 'write midx with twelve packs' '
 
 compare_results_with_midx "twelve packs"
 
+test_expect_success 'verify multi-pack-index success' '
+       git multi-pack-index verify --object-dir=$objdir
+'
+
+# usage: corrupt_midx_and_verify <pos> <data> <objdir> <string>
+corrupt_midx_and_verify() {
+       POS=$1 &&
+       DATA="${2:-\0}" &&
+       OBJDIR=$3 &&
+       GREPSTR="$4" &&
+       COMMAND="$5" &&
+       if test -z "$COMMAND"
+       then
+               COMMAND="git multi-pack-index verify --object-dir=$OBJDIR"
+       fi &&
+       FILE=$OBJDIR/pack/multi-pack-index &&
+       chmod a+w $FILE &&
+       test_when_finished mv midx-backup $FILE &&
+       cp $FILE midx-backup &&
+       printf "$DATA" | dd of="$FILE" bs=1 seek="$POS" conv=notrunc &&
+       test_must_fail $COMMAND 2>test_err &&
+       grep -v "^+" test_err >err &&
+       test_i18ngrep "$GREPSTR" err
+}
+
+test_expect_success 'verify bad signature' '
+       corrupt_midx_and_verify 0 "\00" $objdir \
+               "multi-pack-index signature"
+'
+
+HASH_LEN=20
+NUM_OBJECTS=74
+MIDX_BYTE_VERSION=4
+MIDX_BYTE_OID_VERSION=5
+MIDX_BYTE_CHUNK_COUNT=6
+MIDX_HEADER_SIZE=12
+MIDX_BYTE_CHUNK_ID=$MIDX_HEADER_SIZE
+MIDX_BYTE_CHUNK_OFFSET=$(($MIDX_HEADER_SIZE + 4))
+MIDX_NUM_CHUNKS=5
+MIDX_CHUNK_LOOKUP_WIDTH=12
+MIDX_OFFSET_PACKNAMES=$(($MIDX_HEADER_SIZE + \
+                        $MIDX_NUM_CHUNKS * $MIDX_CHUNK_LOOKUP_WIDTH))
+MIDX_BYTE_PACKNAME_ORDER=$(($MIDX_OFFSET_PACKNAMES + 2))
+MIDX_OFFSET_OID_FANOUT=$(($MIDX_OFFSET_PACKNAMES + 652))
+MIDX_OID_FANOUT_WIDTH=4
+MIDX_BYTE_OID_FANOUT_ORDER=$((MIDX_OFFSET_OID_FANOUT + 250 * $MIDX_OID_FANOUT_WIDTH + 1))
+MIDX_OFFSET_OID_LOOKUP=$(($MIDX_OFFSET_OID_FANOUT + 256 * $MIDX_OID_FANOUT_WIDTH))
+MIDX_BYTE_OID_LOOKUP=$(($MIDX_OFFSET_OID_LOOKUP + 16 * $HASH_LEN))
+MIDX_OFFSET_OBJECT_OFFSETS=$(($MIDX_OFFSET_OID_LOOKUP + $NUM_OBJECTS * $HASH_LEN))
+MIDX_OFFSET_WIDTH=8
+MIDX_BYTE_PACK_INT_ID=$(($MIDX_OFFSET_OBJECT_OFFSETS + 16 * $MIDX_OFFSET_WIDTH + 2))
+MIDX_BYTE_OFFSET=$(($MIDX_OFFSET_OBJECT_OFFSETS + 16 * $MIDX_OFFSET_WIDTH + 6))
+
+test_expect_success 'verify bad version' '
+       corrupt_midx_and_verify $MIDX_BYTE_VERSION "\00" $objdir \
+               "multi-pack-index version"
+'
+
+test_expect_success 'verify bad OID version' '
+       corrupt_midx_and_verify $MIDX_BYTE_OID_VERSION "\02" $objdir \
+               "hash version"
+'
+
+test_expect_success 'verify truncated chunk count' '
+       corrupt_midx_and_verify $MIDX_BYTE_CHUNK_COUNT "\01" $objdir \
+               "missing required"
+'
+
+test_expect_success 'verify extended chunk count' '
+       corrupt_midx_and_verify $MIDX_BYTE_CHUNK_COUNT "\07" $objdir \
+               "terminating multi-pack-index chunk id appears earlier than expected"
+'
+
+test_expect_success 'verify missing required chunk' '
+       corrupt_midx_and_verify $MIDX_BYTE_CHUNK_ID "\01" $objdir \
+               "missing required"
+'
+
+test_expect_success 'verify invalid chunk offset' '
+       corrupt_midx_and_verify $MIDX_BYTE_CHUNK_OFFSET "\01" $objdir \
+               "invalid chunk offset (too large)"
+'
+
+test_expect_success 'verify packnames out of order' '
+       corrupt_midx_and_verify $MIDX_BYTE_PACKNAME_ORDER "z" $objdir \
+               "pack names out of order"
+'
+
+test_expect_success 'verify packnames out of order' '
+       corrupt_midx_and_verify $MIDX_BYTE_PACKNAME_ORDER "a" $objdir \
+               "failed to load pack"
+'
+
+test_expect_success 'verify oid fanout out of order' '
+       corrupt_midx_and_verify $MIDX_BYTE_OID_FANOUT_ORDER "\01" $objdir \
+               "oid fanout out of order"
+'
+
+test_expect_success 'verify oid lookup out of order' '
+       corrupt_midx_and_verify $MIDX_BYTE_OID_LOOKUP "\00" $objdir \
+               "oid lookup out of order"
+'
+
+test_expect_success 'verify incorrect pack-int-id' '
+       corrupt_midx_and_verify $MIDX_BYTE_PACK_INT_ID "\07" $objdir \
+               "bad pack-int-id"
+'
+
+test_expect_success 'verify incorrect offset' '
+       corrupt_midx_and_verify $MIDX_BYTE_OFFSET "\07" $objdir \
+               "incorrect object offset"
+'
+
+test_expect_success 'git-fsck incorrect offset' '
+       corrupt_midx_and_verify $MIDX_BYTE_OFFSET "\07" $objdir \
+               "incorrect object offset" \
+               "git -c core.multipackindex=true fsck"
+'
+
 test_expect_success 'repack removes multi-pack-index' '
        test_path_is_file $objdir/pack/multi-pack-index &&
        git repack -adf &&
@@ -187,7 +306,6 @@ test_expect_success 'multi-pack-index in an alternate' '
 
 compare_results_with_midx "with alternate (remote midx)"
 
-
 # usage: corrupt_data <file> <pos> [<data>]
 corrupt_data () {
        file=$1
@@ -214,4 +332,20 @@ test_expect_success 'force some 64-bit offsets with pack-objects' '
        midx_read_expect 1 63 5 objects64 " large-offsets"
 '
 
+test_expect_success 'verify multi-pack-index with 64-bit offsets' '
+       git multi-pack-index verify --object-dir=objects64
+'
+
+NUM_OBJECTS=63
+MIDX_OFFSET_OID_FANOUT=$((MIDX_OFFSET_PACKNAMES + 54))
+MIDX_OFFSET_OID_LOOKUP=$((MIDX_OFFSET_OID_FANOUT + 256 * $MIDX_OID_FANOUT_WIDTH))
+MIDX_OFFSET_OBJECT_OFFSETS=$(($MIDX_OFFSET_OID_LOOKUP + $NUM_OBJECTS * $HASH_LEN))
+MIDX_OFFSET_LARGE_OFFSETS=$(($MIDX_OFFSET_OBJECT_OFFSETS + $NUM_OBJECTS * $MIDX_OFFSET_WIDTH))
+MIDX_BYTE_LARGE_OFFSET=$(($MIDX_OFFSET_LARGE_OFFSETS + 3))
+
+test_expect_success 'verify incorrect 64-bit offset' '
+       corrupt_midx_and_verify $MIDX_BYTE_LARGE_OFFSET "\07" objects64 \
+               "incorrect object offset"
+'
+
 test_done