multi-pack-index: read packfile list
[git/git.git] / midx.c
CommitLineData
a3407730 1#include "cache.h"
fc59e748 2#include "csum-file.h"
396f2570 3#include "dir.h"
fc59e748 4#include "lockfile.h"
396f2570 5#include "packfile.h"
4d80560c 6#include "object-store.h"
a3407730
DS
7#include "midx.h"
8
fc59e748
DS
9#define MIDX_SIGNATURE 0x4d494458 /* "MIDX" */
10#define MIDX_VERSION 1
4d80560c
DS
11#define MIDX_BYTE_FILE_VERSION 4
12#define MIDX_BYTE_HASH_VERSION 5
13#define MIDX_BYTE_NUM_CHUNKS 6
14#define MIDX_BYTE_NUM_PACKS 8
fc59e748
DS
15#define MIDX_HASH_VERSION 1
16#define MIDX_HEADER_SIZE 12
4d80560c
DS
17#define MIDX_HASH_LEN 20
18#define MIDX_MIN_SIZE (MIDX_HEADER_SIZE + MIDX_HASH_LEN)
fc59e748
DS
19
20static char *get_midx_filename(const char *object_dir)
21{
22 return xstrfmt("%s/pack/multi-pack-index", object_dir);
23}
24
4d80560c
DS
25struct multi_pack_index *load_multi_pack_index(const char *object_dir)
26{
27 struct multi_pack_index *m = NULL;
28 int fd;
29 struct stat st;
30 size_t midx_size;
31 void *midx_map = NULL;
32 uint32_t hash_version;
33 char *midx_name = get_midx_filename(object_dir);
34
35 fd = git_open(midx_name);
36
37 if (fd < 0)
38 goto cleanup_fail;
39 if (fstat(fd, &st)) {
40 error_errno(_("failed to read %s"), midx_name);
41 goto cleanup_fail;
42 }
43
44 midx_size = xsize_t(st.st_size);
45
46 if (midx_size < MIDX_MIN_SIZE) {
47 error(_("multi-pack-index file %s is too small"), midx_name);
48 goto cleanup_fail;
49 }
50
51 FREE_AND_NULL(midx_name);
52
53 midx_map = xmmap(NULL, midx_size, PROT_READ, MAP_PRIVATE, fd, 0);
54
55 FLEX_ALLOC_MEM(m, object_dir, object_dir, strlen(object_dir));
56 m->fd = fd;
57 m->data = midx_map;
58 m->data_len = midx_size;
59
60 m->signature = get_be32(m->data);
61 if (m->signature != MIDX_SIGNATURE) {
62 error(_("multi-pack-index signature 0x%08x does not match signature 0x%08x"),
63 m->signature, MIDX_SIGNATURE);
64 goto cleanup_fail;
65 }
66
67 m->version = m->data[MIDX_BYTE_FILE_VERSION];
68 if (m->version != MIDX_VERSION) {
69 error(_("multi-pack-index version %d not recognized"),
70 m->version);
71 goto cleanup_fail;
72 }
73
74 hash_version = m->data[MIDX_BYTE_HASH_VERSION];
75 if (hash_version != MIDX_HASH_VERSION) {
76 error(_("hash version %u does not match"), hash_version);
77 goto cleanup_fail;
78 }
79 m->hash_len = MIDX_HASH_LEN;
80
81 m->num_chunks = m->data[MIDX_BYTE_NUM_CHUNKS];
82
83 m->num_packs = get_be32(m->data + MIDX_BYTE_NUM_PACKS);
84
85 return m;
86
87cleanup_fail:
88 free(m);
89 free(midx_name);
90 if (midx_map)
91 munmap(midx_map, midx_size);
92 if (0 <= fd)
93 close(fd);
94 return NULL;
95}
96
fc59e748
DS
97static size_t write_midx_header(struct hashfile *f,
98 unsigned char num_chunks,
99 uint32_t num_packs)
100{
101 unsigned char byte_values[4];
102
103 hashwrite_be32(f, MIDX_SIGNATURE);
104 byte_values[0] = MIDX_VERSION;
105 byte_values[1] = MIDX_HASH_VERSION;
106 byte_values[2] = num_chunks;
107 byte_values[3] = 0; /* unused */
108 hashwrite(f, byte_values, sizeof(byte_values));
109 hashwrite_be32(f, num_packs);
110
111 return MIDX_HEADER_SIZE;
112}
113
396f2570
DS
114struct pack_list {
115 struct packed_git **list;
116 uint32_t nr;
117 uint32_t alloc_list;
118};
119
120static void add_pack_to_midx(const char *full_path, size_t full_path_len,
121 const char *file_name, void *data)
122{
123 struct pack_list *packs = (struct pack_list *)data;
124
125 if (ends_with(file_name, ".idx")) {
126 ALLOC_GROW(packs->list, packs->nr + 1, packs->alloc_list);
127
128 packs->list[packs->nr] = add_packed_git(full_path,
129 full_path_len,
130 0);
131 if (!packs->list[packs->nr]) {
132 warning(_("failed to add packfile '%s'"),
133 full_path);
134 return;
135 }
136
137 packs->nr++;
138 }
139}
140
a3407730
DS
141int write_midx_file(const char *object_dir)
142{
fc59e748
DS
143 unsigned char num_chunks = 0;
144 char *midx_name;
396f2570 145 uint32_t i;
fc59e748
DS
146 struct hashfile *f = NULL;
147 struct lock_file lk;
396f2570 148 struct pack_list packs;
fc59e748
DS
149
150 midx_name = get_midx_filename(object_dir);
151 if (safe_create_leading_directories(midx_name)) {
152 UNLEAK(midx_name);
153 die_errno(_("unable to create leading directories of %s"),
154 midx_name);
155 }
156
396f2570
DS
157 packs.nr = 0;
158 packs.alloc_list = 16;
159 packs.list = NULL;
160 ALLOC_ARRAY(packs.list, packs.alloc_list);
161
162 for_each_file_in_pack_dir(object_dir, add_pack_to_midx, &packs);
163
fc59e748
DS
164 hold_lock_file_for_update(&lk, midx_name, LOCK_DIE_ON_ERROR);
165 f = hashfd(lk.tempfile->fd, lk.tempfile->filename.buf);
166 FREE_AND_NULL(midx_name);
167
396f2570 168 write_midx_header(f, num_chunks, packs.nr);
fc59e748
DS
169
170 finalize_hashfile(f, NULL, CSUM_FSYNC | CSUM_HASH_IN_STREAM);
171 commit_lock_file(&lk);
172
396f2570
DS
173 for (i = 0; i < packs.nr; i++) {
174 if (packs.list[i]) {
175 close_pack(packs.list[i]);
176 free(packs.list[i]);
177 }
178 }
179
180 free(packs.list);
a3407730
DS
181 return 0;
182}