promisor-remote: skip move_to_tail when no-op
[git/git.git] / name-hash.c
1 /*
2 * name-hash.c
3 *
4 * Hashing names in the index state
5 *
6 * Copyright (C) 2008 Linus Torvalds
7 */
8 #include "cache.h"
9 #include "thread-utils.h"
10
11 struct dir_entry {
12 struct hashmap_entry ent;
13 struct dir_entry *parent;
14 int nr;
15 unsigned int namelen;
16 char name[FLEX_ARRAY];
17 };
18
19 static int dir_entry_cmp(const void *unused_cmp_data,
20 const void *entry,
21 const void *entry_or_key,
22 const void *keydata)
23 {
24 const struct dir_entry *e1 = entry;
25 const struct dir_entry *e2 = entry_or_key;
26 const char *name = keydata;
27
28 return e1->namelen != e2->namelen || strncasecmp(e1->name,
29 name ? name : e2->name, e1->namelen);
30 }
31
32 static struct dir_entry *find_dir_entry__hash(struct index_state *istate,
33 const char *name, unsigned int namelen, unsigned int hash)
34 {
35 struct dir_entry key;
36 hashmap_entry_init(&key, hash);
37 key.namelen = namelen;
38 return hashmap_get(&istate->dir_hash, &key, name);
39 }
40
41 static struct dir_entry *find_dir_entry(struct index_state *istate,
42 const char *name, unsigned int namelen)
43 {
44 return find_dir_entry__hash(istate, name, namelen, memihash(name, namelen));
45 }
46
47 static struct dir_entry *hash_dir_entry(struct index_state *istate,
48 struct cache_entry *ce, int namelen)
49 {
50 /*
51 * Throw each directory component in the hash for quick lookup
52 * during a git status. Directory components are stored without their
53 * closing slash. Despite submodules being a directory, they never
54 * reach this point, because they are stored
55 * in index_state.name_hash (as ordinary cache_entries).
56 */
57 struct dir_entry *dir;
58
59 /* get length of parent directory */
60 while (namelen > 0 && !is_dir_sep(ce->name[namelen - 1]))
61 namelen--;
62 if (namelen <= 0)
63 return NULL;
64 namelen--;
65
66 /* lookup existing entry for that directory */
67 dir = find_dir_entry(istate, ce->name, namelen);
68 if (!dir) {
69 /* not found, create it and add to hash table */
70 FLEX_ALLOC_MEM(dir, name, ce->name, namelen);
71 hashmap_entry_init(dir, memihash(ce->name, namelen));
72 dir->namelen = namelen;
73 hashmap_add(&istate->dir_hash, dir);
74
75 /* recursively add missing parent directories */
76 dir->parent = hash_dir_entry(istate, ce, namelen);
77 }
78 return dir;
79 }
80
81 static void add_dir_entry(struct index_state *istate, struct cache_entry *ce)
82 {
83 /* Add reference to the directory entry (and parents if 0). */
84 struct dir_entry *dir = hash_dir_entry(istate, ce, ce_namelen(ce));
85 while (dir && !(dir->nr++))
86 dir = dir->parent;
87 }
88
89 static void remove_dir_entry(struct index_state *istate, struct cache_entry *ce)
90 {
91 /*
92 * Release reference to the directory entry. If 0, remove and continue
93 * with parent directory.
94 */
95 struct dir_entry *dir = hash_dir_entry(istate, ce, ce_namelen(ce));
96 while (dir && !(--dir->nr)) {
97 struct dir_entry *parent = dir->parent;
98 hashmap_remove(&istate->dir_hash, dir, NULL);
99 free(dir);
100 dir = parent;
101 }
102 }
103
104 static void hash_index_entry(struct index_state *istate, struct cache_entry *ce)
105 {
106 if (ce->ce_flags & CE_HASHED)
107 return;
108 ce->ce_flags |= CE_HASHED;
109 hashmap_entry_init(ce, memihash(ce->name, ce_namelen(ce)));
110 hashmap_add(&istate->name_hash, ce);
111
112 if (ignore_case)
113 add_dir_entry(istate, ce);
114 }
115
116 static int cache_entry_cmp(const void *unused_cmp_data,
117 const void *entry,
118 const void *entry_or_key,
119 const void *remove)
120 {
121 const struct cache_entry *ce1 = entry;
122 const struct cache_entry *ce2 = entry_or_key;
123 /*
124 * For remove_name_hash, find the exact entry (pointer equality); for
125 * index_file_exists, find all entries with matching hash code and
126 * decide whether the entry matches in same_name.
127 */
128 return remove ? !(ce1 == ce2) : 0;
129 }
130
131 static int lazy_try_threaded = 1;
132 static int lazy_nr_dir_threads;
133
134 /*
135 * Set a minimum number of cache_entries that we will handle per
136 * thread and use that to decide how many threads to run (upto
137 * the number on the system).
138 *
139 * For guidance setting the lower per-thread bound, see:
140 * t/helper/test-lazy-init-name-hash --analyze
141 */
142 #define LAZY_THREAD_COST (2000)
143
144 /*
145 * We use n mutexes to guard n partitions of the "istate->dir_hash"
146 * hashtable. Since "find" and "insert" operations will hash to a
147 * particular bucket and modify/search a single chain, we can say
148 * that "all chains mod n" are guarded by the same mutex -- rather
149 * than having a single mutex to guard the entire table. (This does
150 * require that we disable "rehashing" on the hashtable.)
151 *
152 * So, a larger value here decreases the probability of a collision
153 * and the time that each thread must wait for the mutex.
154 */
155 #define LAZY_MAX_MUTEX (32)
156
157 static pthread_mutex_t *lazy_dir_mutex_array;
158
159 /*
160 * An array of lazy_entry items is used by the n threads in
161 * the directory parse (first) phase to (lock-free) store the
162 * intermediate results. These values are then referenced by
163 * the 2 threads in the second phase.
164 */
165 struct lazy_entry {
166 struct dir_entry *dir;
167 unsigned int hash_dir;
168 unsigned int hash_name;
169 };
170
171 /*
172 * Decide if we want to use threads (if available) to load
173 * the hash tables. We set "lazy_nr_dir_threads" to zero when
174 * it is not worth it.
175 */
176 static int lookup_lazy_params(struct index_state *istate)
177 {
178 int nr_cpus;
179
180 lazy_nr_dir_threads = 0;
181
182 if (!lazy_try_threaded)
183 return 0;
184
185 /*
186 * If we are respecting case, just use the original
187 * code to build the "istate->name_hash". We don't
188 * need the complexity here.
189 */
190 if (!ignore_case)
191 return 0;
192
193 nr_cpus = online_cpus();
194 if (nr_cpus < 2)
195 return 0;
196
197 if (istate->cache_nr < 2 * LAZY_THREAD_COST)
198 return 0;
199
200 if (istate->cache_nr < nr_cpus * LAZY_THREAD_COST)
201 nr_cpus = istate->cache_nr / LAZY_THREAD_COST;
202 lazy_nr_dir_threads = nr_cpus;
203 return lazy_nr_dir_threads;
204 }
205
206 /*
207 * Initialize n mutexes for use when searching and inserting
208 * into "istate->dir_hash". All "dir" threads are trying
209 * to insert partial pathnames into the hash as they iterate
210 * over their portions of the index, so lock contention is
211 * high.
212 *
213 * However, the hashmap is going to put items into bucket
214 * chains based on their hash values. Use that to create n
215 * mutexes and lock on mutex[bucket(hash) % n]. This will
216 * decrease the collision rate by (hopefully) by a factor of n.
217 */
218 static void init_dir_mutex(void)
219 {
220 int j;
221
222 lazy_dir_mutex_array = xcalloc(LAZY_MAX_MUTEX, sizeof(pthread_mutex_t));
223
224 for (j = 0; j < LAZY_MAX_MUTEX; j++)
225 init_recursive_mutex(&lazy_dir_mutex_array[j]);
226 }
227
228 static void cleanup_dir_mutex(void)
229 {
230 int j;
231
232 for (j = 0; j < LAZY_MAX_MUTEX; j++)
233 pthread_mutex_destroy(&lazy_dir_mutex_array[j]);
234
235 free(lazy_dir_mutex_array);
236 }
237
238 static void lock_dir_mutex(int j)
239 {
240 pthread_mutex_lock(&lazy_dir_mutex_array[j]);
241 }
242
243 static void unlock_dir_mutex(int j)
244 {
245 pthread_mutex_unlock(&lazy_dir_mutex_array[j]);
246 }
247
248 static inline int compute_dir_lock_nr(
249 const struct hashmap *map,
250 unsigned int hash)
251 {
252 return hashmap_bucket(map, hash) % LAZY_MAX_MUTEX;
253 }
254
255 static struct dir_entry *hash_dir_entry_with_parent_and_prefix(
256 struct index_state *istate,
257 struct dir_entry *parent,
258 struct strbuf *prefix)
259 {
260 struct dir_entry *dir;
261 unsigned int hash;
262 int lock_nr;
263
264 /*
265 * Either we have a parent directory and path with slash(es)
266 * or the directory is an immediate child of the root directory.
267 */
268 assert((parent != NULL) ^ (strchr(prefix->buf, '/') == NULL));
269
270 if (parent)
271 hash = memihash_cont(parent->ent.hash,
272 prefix->buf + parent->namelen,
273 prefix->len - parent->namelen);
274 else
275 hash = memihash(prefix->buf, prefix->len);
276
277 lock_nr = compute_dir_lock_nr(&istate->dir_hash, hash);
278 lock_dir_mutex(lock_nr);
279
280 dir = find_dir_entry__hash(istate, prefix->buf, prefix->len, hash);
281 if (!dir) {
282 FLEX_ALLOC_MEM(dir, name, prefix->buf, prefix->len);
283 hashmap_entry_init(dir, hash);
284 dir->namelen = prefix->len;
285 dir->parent = parent;
286 hashmap_add(&istate->dir_hash, dir);
287
288 if (parent) {
289 unlock_dir_mutex(lock_nr);
290
291 /* All I really need here is an InterlockedIncrement(&(parent->nr)) */
292 lock_nr = compute_dir_lock_nr(&istate->dir_hash, parent->ent.hash);
293 lock_dir_mutex(lock_nr);
294 parent->nr++;
295 }
296 }
297
298 unlock_dir_mutex(lock_nr);
299
300 return dir;
301 }
302
303 /*
304 * handle_range_1() and handle_range_dir() are derived from
305 * clear_ce_flags_1() and clear_ce_flags_dir() in unpack-trees.c
306 * and handle the iteration over the entire array of index entries.
307 * They use recursion for adjacent entries in the same parent
308 * directory.
309 */
310 static int handle_range_1(
311 struct index_state *istate,
312 int k_start,
313 int k_end,
314 struct dir_entry *parent,
315 struct strbuf *prefix,
316 struct lazy_entry *lazy_entries);
317
318 static int handle_range_dir(
319 struct index_state *istate,
320 int k_start,
321 int k_end,
322 struct dir_entry *parent,
323 struct strbuf *prefix,
324 struct lazy_entry *lazy_entries,
325 struct dir_entry **dir_new_out)
326 {
327 int rc, k;
328 int input_prefix_len = prefix->len;
329 struct dir_entry *dir_new;
330
331 dir_new = hash_dir_entry_with_parent_and_prefix(istate, parent, prefix);
332
333 strbuf_addch(prefix, '/');
334
335 /*
336 * Scan forward in the index array for index entries having the same
337 * path prefix (that are also in this directory).
338 */
339 if (k_start + 1 >= k_end)
340 k = k_end;
341 else if (strncmp(istate->cache[k_start + 1]->name, prefix->buf, prefix->len) > 0)
342 k = k_start + 1;
343 else if (strncmp(istate->cache[k_end - 1]->name, prefix->buf, prefix->len) == 0)
344 k = k_end;
345 else {
346 int begin = k_start;
347 int end = k_end;
348 while (begin < end) {
349 int mid = (begin + end) >> 1;
350 int cmp = strncmp(istate->cache[mid]->name, prefix->buf, prefix->len);
351 if (cmp == 0) /* mid has same prefix; look in second part */
352 begin = mid + 1;
353 else if (cmp > 0) /* mid is past group; look in first part */
354 end = mid;
355 else
356 die("cache entry out of order");
357 }
358 k = begin;
359 }
360
361 /*
362 * Recurse and process what we can of this subset [k_start, k).
363 */
364 rc = handle_range_1(istate, k_start, k, dir_new, prefix, lazy_entries);
365
366 strbuf_setlen(prefix, input_prefix_len);
367
368 *dir_new_out = dir_new;
369 return rc;
370 }
371
372 static int handle_range_1(
373 struct index_state *istate,
374 int k_start,
375 int k_end,
376 struct dir_entry *parent,
377 struct strbuf *prefix,
378 struct lazy_entry *lazy_entries)
379 {
380 int input_prefix_len = prefix->len;
381 int k = k_start;
382
383 while (k < k_end) {
384 struct cache_entry *ce_k = istate->cache[k];
385 const char *name, *slash;
386
387 if (prefix->len && strncmp(ce_k->name, prefix->buf, prefix->len))
388 break;
389
390 name = ce_k->name + prefix->len;
391 slash = strchr(name, '/');
392
393 if (slash) {
394 int len = slash - name;
395 int processed;
396 struct dir_entry *dir_new;
397
398 strbuf_add(prefix, name, len);
399 processed = handle_range_dir(istate, k, k_end, parent, prefix, lazy_entries, &dir_new);
400 if (processed) {
401 k += processed;
402 strbuf_setlen(prefix, input_prefix_len);
403 continue;
404 }
405
406 strbuf_addch(prefix, '/');
407 processed = handle_range_1(istate, k, k_end, dir_new, prefix, lazy_entries);
408 k += processed;
409 strbuf_setlen(prefix, input_prefix_len);
410 continue;
411 }
412
413 /*
414 * It is too expensive to take a lock to insert "ce_k"
415 * into "istate->name_hash" and increment the ref-count
416 * on the "parent" dir. So we defer actually updating
417 * permanent data structures until phase 2 (where we
418 * can change the locking requirements) and simply
419 * accumulate our current results into the lazy_entries
420 * data array).
421 *
422 * We do not need to lock the lazy_entries array because
423 * we have exclusive access to the cells in the range
424 * [k_start,k_end) that this thread was given.
425 */
426 lazy_entries[k].dir = parent;
427 if (parent) {
428 lazy_entries[k].hash_name = memihash_cont(
429 parent->ent.hash,
430 ce_k->name + parent->namelen,
431 ce_namelen(ce_k) - parent->namelen);
432 lazy_entries[k].hash_dir = parent->ent.hash;
433 } else {
434 lazy_entries[k].hash_name = memihash(ce_k->name, ce_namelen(ce_k));
435 }
436
437 k++;
438 }
439
440 return k - k_start;
441 }
442
443 struct lazy_dir_thread_data {
444 pthread_t pthread;
445 struct index_state *istate;
446 struct lazy_entry *lazy_entries;
447 int k_start;
448 int k_end;
449 };
450
451 static void *lazy_dir_thread_proc(void *_data)
452 {
453 struct lazy_dir_thread_data *d = _data;
454 struct strbuf prefix = STRBUF_INIT;
455 handle_range_1(d->istate, d->k_start, d->k_end, NULL, &prefix, d->lazy_entries);
456 strbuf_release(&prefix);
457 return NULL;
458 }
459
460 struct lazy_name_thread_data {
461 pthread_t pthread;
462 struct index_state *istate;
463 struct lazy_entry *lazy_entries;
464 };
465
466 static void *lazy_name_thread_proc(void *_data)
467 {
468 struct lazy_name_thread_data *d = _data;
469 int k;
470
471 for (k = 0; k < d->istate->cache_nr; k++) {
472 struct cache_entry *ce_k = d->istate->cache[k];
473 ce_k->ce_flags |= CE_HASHED;
474 hashmap_entry_init(ce_k, d->lazy_entries[k].hash_name);
475 hashmap_add(&d->istate->name_hash, ce_k);
476 }
477
478 return NULL;
479 }
480
481 static inline void lazy_update_dir_ref_counts(
482 struct index_state *istate,
483 struct lazy_entry *lazy_entries)
484 {
485 int k;
486
487 for (k = 0; k < istate->cache_nr; k++) {
488 if (lazy_entries[k].dir)
489 lazy_entries[k].dir->nr++;
490 }
491 }
492
493 static void threaded_lazy_init_name_hash(
494 struct index_state *istate)
495 {
496 int err;
497 int nr_each;
498 int k_start;
499 int t;
500 struct lazy_entry *lazy_entries;
501 struct lazy_dir_thread_data *td_dir;
502 struct lazy_name_thread_data *td_name;
503
504 if (!HAVE_THREADS)
505 return;
506
507 k_start = 0;
508 nr_each = DIV_ROUND_UP(istate->cache_nr, lazy_nr_dir_threads);
509
510 lazy_entries = xcalloc(istate->cache_nr, sizeof(struct lazy_entry));
511 td_dir = xcalloc(lazy_nr_dir_threads, sizeof(struct lazy_dir_thread_data));
512 td_name = xcalloc(1, sizeof(struct lazy_name_thread_data));
513
514 init_dir_mutex();
515
516 /*
517 * Phase 1:
518 * Build "istate->dir_hash" using n "dir" threads (and a read-only index).
519 */
520 for (t = 0; t < lazy_nr_dir_threads; t++) {
521 struct lazy_dir_thread_data *td_dir_t = td_dir + t;
522 td_dir_t->istate = istate;
523 td_dir_t->lazy_entries = lazy_entries;
524 td_dir_t->k_start = k_start;
525 k_start += nr_each;
526 if (k_start > istate->cache_nr)
527 k_start = istate->cache_nr;
528 td_dir_t->k_end = k_start;
529 err = pthread_create(&td_dir_t->pthread, NULL, lazy_dir_thread_proc, td_dir_t);
530 if (err)
531 die(_("unable to create lazy_dir thread: %s"), strerror(err));
532 }
533 for (t = 0; t < lazy_nr_dir_threads; t++) {
534 struct lazy_dir_thread_data *td_dir_t = td_dir + t;
535 if (pthread_join(td_dir_t->pthread, NULL))
536 die("unable to join lazy_dir_thread");
537 }
538
539 /*
540 * Phase 2:
541 * Iterate over all index entries and add them to the "istate->name_hash"
542 * using a single "name" background thread.
543 * (Testing showed it wasn't worth running more than 1 thread for this.)
544 *
545 * Meanwhile, finish updating the parent directory ref-counts for each
546 * index entry using the current thread. (This step is very fast and
547 * doesn't need threading.)
548 */
549 td_name->istate = istate;
550 td_name->lazy_entries = lazy_entries;
551 err = pthread_create(&td_name->pthread, NULL, lazy_name_thread_proc, td_name);
552 if (err)
553 die(_("unable to create lazy_name thread: %s"), strerror(err));
554
555 lazy_update_dir_ref_counts(istate, lazy_entries);
556
557 err = pthread_join(td_name->pthread, NULL);
558 if (err)
559 die(_("unable to join lazy_name thread: %s"), strerror(err));
560
561 cleanup_dir_mutex();
562
563 free(td_name);
564 free(td_dir);
565 free(lazy_entries);
566 }
567
568 static void lazy_init_name_hash(struct index_state *istate)
569 {
570
571 if (istate->name_hash_initialized)
572 return;
573 trace_performance_enter();
574 hashmap_init(&istate->name_hash, cache_entry_cmp, NULL, istate->cache_nr);
575 hashmap_init(&istate->dir_hash, dir_entry_cmp, NULL, istate->cache_nr);
576
577 if (lookup_lazy_params(istate)) {
578 /*
579 * Disable item counting and automatic rehashing because
580 * we do per-chain (mod n) locking rather than whole hashmap
581 * locking and we need to prevent the table-size from changing
582 * and bucket items from being redistributed.
583 */
584 hashmap_disable_item_counting(&istate->dir_hash);
585 threaded_lazy_init_name_hash(istate);
586 hashmap_enable_item_counting(&istate->dir_hash);
587 } else {
588 int nr;
589 for (nr = 0; nr < istate->cache_nr; nr++)
590 hash_index_entry(istate, istate->cache[nr]);
591 }
592
593 istate->name_hash_initialized = 1;
594 trace_performance_leave("initialize name hash");
595 }
596
597 /*
598 * A test routine for t/helper/ sources.
599 *
600 * Returns the number of threads used or 0 when
601 * the non-threaded code path was used.
602 *
603 * Requesting threading WILL NOT override guards
604 * in lookup_lazy_params().
605 */
606 int test_lazy_init_name_hash(struct index_state *istate, int try_threaded)
607 {
608 lazy_nr_dir_threads = 0;
609 lazy_try_threaded = try_threaded;
610
611 lazy_init_name_hash(istate);
612
613 return lazy_nr_dir_threads;
614 }
615
616 void add_name_hash(struct index_state *istate, struct cache_entry *ce)
617 {
618 if (istate->name_hash_initialized)
619 hash_index_entry(istate, ce);
620 }
621
622 void remove_name_hash(struct index_state *istate, struct cache_entry *ce)
623 {
624 if (!istate->name_hash_initialized || !(ce->ce_flags & CE_HASHED))
625 return;
626 ce->ce_flags &= ~CE_HASHED;
627 hashmap_remove(&istate->name_hash, ce, ce);
628
629 if (ignore_case)
630 remove_dir_entry(istate, ce);
631 }
632
633 static int slow_same_name(const char *name1, int len1, const char *name2, int len2)
634 {
635 if (len1 != len2)
636 return 0;
637
638 while (len1) {
639 unsigned char c1 = *name1++;
640 unsigned char c2 = *name2++;
641 len1--;
642 if (c1 != c2) {
643 c1 = toupper(c1);
644 c2 = toupper(c2);
645 if (c1 != c2)
646 return 0;
647 }
648 }
649 return 1;
650 }
651
652 static int same_name(const struct cache_entry *ce, const char *name, int namelen, int icase)
653 {
654 int len = ce_namelen(ce);
655
656 /*
657 * Always do exact compare, even if we want a case-ignoring comparison;
658 * we do the quick exact one first, because it will be the common case.
659 */
660 if (len == namelen && !memcmp(name, ce->name, len))
661 return 1;
662
663 if (!icase)
664 return 0;
665
666 return slow_same_name(name, namelen, ce->name, len);
667 }
668
669 int index_dir_exists(struct index_state *istate, const char *name, int namelen)
670 {
671 struct dir_entry *dir;
672
673 lazy_init_name_hash(istate);
674 dir = find_dir_entry(istate, name, namelen);
675 return dir && dir->nr;
676 }
677
678 void adjust_dirname_case(struct index_state *istate, char *name)
679 {
680 const char *startPtr = name;
681 const char *ptr = startPtr;
682
683 lazy_init_name_hash(istate);
684 while (*ptr) {
685 while (*ptr && *ptr != '/')
686 ptr++;
687
688 if (*ptr == '/') {
689 struct dir_entry *dir;
690
691 dir = find_dir_entry(istate, name, ptr - name);
692 if (dir) {
693 memcpy((void *)startPtr, dir->name + (startPtr - name), ptr - startPtr);
694 startPtr = ptr + 1;
695 }
696 ptr++;
697 }
698 }
699 }
700
701 struct cache_entry *index_file_exists(struct index_state *istate, const char *name, int namelen, int icase)
702 {
703 struct cache_entry *ce;
704
705 lazy_init_name_hash(istate);
706
707 ce = hashmap_get_from_hash(&istate->name_hash,
708 memihash(name, namelen), NULL);
709 while (ce) {
710 if (same_name(ce, name, namelen, icase))
711 return ce;
712 ce = hashmap_get_next(&istate->name_hash, ce);
713 }
714 return NULL;
715 }
716
717 void free_name_hash(struct index_state *istate)
718 {
719 if (!istate->name_hash_initialized)
720 return;
721 istate->name_hash_initialized = 0;
722
723 hashmap_free(&istate->name_hash, 0);
724 hashmap_free(&istate->dir_hash, 1);
725 }