1
2
3
4
5
6
7
8
9
10
11 package org.eclipse.jgit.internal.storage.file;
12
13 import static org.eclipse.jgit.internal.storage.pack.PackWriter.NONE;
14 import static org.eclipse.jgit.lib.Constants.INFO_ALTERNATES;
15 import static org.eclipse.jgit.lib.Constants.OBJ_BLOB;
16 import static org.junit.Assert.assertEquals;
17 import static org.junit.Assert.assertFalse;
18 import static org.junit.Assert.assertNotNull;
19 import static org.junit.Assert.assertTrue;
20 import static org.junit.Assert.fail;
21 import static org.mockito.ArgumentMatchers.any;
22 import static org.mockito.Mockito.doNothing;
23 import static org.mockito.Mockito.times;
24 import static org.mockito.Mockito.verify;
25
26 import java.io.ByteArrayInputStream;
27 import java.io.ByteArrayOutputStream;
28 import java.io.File;
29 import java.io.FileOutputStream;
30 import java.io.IOException;
31 import java.time.Duration;
32 import java.util.ArrayList;
33 import java.util.Arrays;
34 import java.util.Collections;
35 import java.util.HashSet;
36 import java.util.List;
37 import java.util.Set;
38
39 import org.eclipse.jgit.api.Git;
40 import org.eclipse.jgit.errors.MissingObjectException;
41 import org.eclipse.jgit.internal.storage.file.PackIndex.MutableEntry;
42 import org.eclipse.jgit.internal.storage.pack.PackExt;
43 import org.eclipse.jgit.internal.storage.pack.PackWriter;
44 import org.eclipse.jgit.junit.JGitTestUtil;
45 import org.eclipse.jgit.junit.TestRepository;
46 import org.eclipse.jgit.junit.TestRepository.BranchBuilder;
47 import org.eclipse.jgit.lib.NullProgressMonitor;
48 import org.eclipse.jgit.lib.ObjectId;
49 import org.eclipse.jgit.lib.ObjectIdSet;
50 import org.eclipse.jgit.lib.ObjectInserter;
51 import org.eclipse.jgit.lib.Ref;
52 import org.eclipse.jgit.lib.Repository;
53 import org.eclipse.jgit.lib.Sets;
54 import org.eclipse.jgit.revwalk.DepthWalk;
55 import org.eclipse.jgit.revwalk.ObjectWalk;
56 import org.eclipse.jgit.revwalk.RevBlob;
57 import org.eclipse.jgit.revwalk.RevCommit;
58 import org.eclipse.jgit.revwalk.RevObject;
59 import org.eclipse.jgit.revwalk.RevWalk;
60 import org.eclipse.jgit.storage.pack.PackConfig;
61 import org.eclipse.jgit.storage.pack.PackStatistics;
62 import org.eclipse.jgit.test.resources.SampleDataRepositoryTestCase;
63 import org.eclipse.jgit.transport.PackParser;
64 import org.junit.After;
65 import org.junit.Before;
66 import org.junit.Test;
67 import org.mockito.Mockito;
68
69 public class PackWriterTest extends SampleDataRepositoryTestCase {
70
71 private static final List<RevObject> EMPTY_LIST_REVS = Collections
72 .<RevObject> emptyList();
73
74 private static final Set<ObjectIdSet> EMPTY_ID_SET = Collections
75 .<ObjectIdSet> emptySet();
76
77 private PackConfig config;
78
79 private PackWriter writer;
80
81 private ByteArrayOutputStream os;
82
83 private Pack pack;
84
85 private ObjectInserter inserter;
86
87 private FileRepository dst;
88
89 private RevBlob contentA;
90
91 private RevBlob contentB;
92
93 private RevBlob contentC;
94
95 private RevBlob contentD;
96
97 private RevBlob contentE;
98
99 private RevCommit c1;
100
101 private RevCommit c2;
102
103 private RevCommit c3;
104
105 private RevCommit c4;
106
107 private RevCommit c5;
108
109 @Override
110 @Before
111 public void setUp() throws Exception {
112 super.setUp();
113 os = new ByteArrayOutputStream();
114 config = new PackConfig(db);
115
116 dst = createBareRepository();
117 File alt = new File(dst.getObjectDatabase().getDirectory(), INFO_ALTERNATES);
118 alt.getParentFile().mkdirs();
119 write(alt, db.getObjectDatabase().getDirectory().getAbsolutePath() + "\n");
120 }
121
122 @Override
123 @After
124 public void tearDown() throws Exception {
125 if (writer != null) {
126 writer.close();
127 writer = null;
128 }
129 if (inserter != null) {
130 inserter.close();
131 inserter = null;
132 }
133 super.tearDown();
134 }
135
136
137
138
139
140
141 @Test
142 public void testContructor() throws IOException {
143 writer = new PackWriter(config, db.newObjectReader());
144 assertFalse(writer.isDeltaBaseAsOffset());
145 assertTrue(config.isReuseDeltas());
146 assertTrue(config.isReuseObjects());
147 assertEquals(0, writer.getObjectCount());
148 }
149
150
151
152
153 @Test
154 public void testModifySettings() {
155 config.setReuseDeltas(false);
156 config.setReuseObjects(false);
157 config.setDeltaBaseAsOffset(false);
158 assertFalse(config.isReuseDeltas());
159 assertFalse(config.isReuseObjects());
160 assertFalse(config.isDeltaBaseAsOffset());
161
162 writer = new PackWriter(config, db.newObjectReader());
163 writer.setDeltaBaseAsOffset(true);
164 assertTrue(writer.isDeltaBaseAsOffset());
165 assertFalse(config.isDeltaBaseAsOffset());
166 }
167
168
169
170
171
172
173
174 @Test
175 public void testWriteEmptyPack1() throws IOException {
176 createVerifyOpenPack(NONE, NONE, false, false);
177
178 assertEquals(0, writer.getObjectCount());
179 assertEquals(0, pack.getObjectCount());
180 assertEquals("da39a3ee5e6b4b0d3255bfef95601890afd80709", writer
181 .computeName().name());
182 }
183
184
185
186
187
188
189
190 @Test
191 public void testWriteEmptyPack2() throws IOException {
192 createVerifyOpenPack(EMPTY_LIST_REVS);
193
194 assertEquals(0, writer.getObjectCount());
195 assertEquals(0, pack.getObjectCount());
196 }
197
198
199
200
201
202
203
204 @Test
205 public void testNotIgnoreNonExistingObjects() throws IOException {
206 final ObjectId nonExisting = ObjectId
207 .fromString("0000000000000000000000000000000000000001");
208 try {
209 createVerifyOpenPack(NONE, haves(nonExisting), false, false);
210 fail("Should have thrown MissingObjectException");
211 } catch (MissingObjectException x) {
212
213 }
214 }
215
216
217
218
219
220
221 @Test
222 public void testIgnoreNonExistingObjects() throws IOException {
223 final ObjectId nonExisting = ObjectId
224 .fromString("0000000000000000000000000000000000000001");
225 createVerifyOpenPack(NONE, haves(nonExisting), false, true);
226
227 }
228
229
230
231
232
233
234
235
236 @Test
237 public void testIgnoreNonExistingObjectsWithBitmaps() throws Exception {
238 final ObjectId nonExisting = ObjectId
239 .fromString("0000000000000000000000000000000000000001");
240 new GC(db).gc().get();
241 createVerifyOpenPack(NONE, haves(nonExisting), false, true, true);
242
243 }
244
245
246
247
248
249
250
251 @Test
252 public void testWritePack1() throws IOException {
253 config.setReuseDeltas(false);
254 writeVerifyPack1();
255 }
256
257
258
259
260
261
262
263 @Test
264 public void testWritePack1NoObjectReuse() throws IOException {
265 config.setReuseDeltas(false);
266 config.setReuseObjects(false);
267 writeVerifyPack1();
268 }
269
270
271
272
273
274
275
276 @Test
277 public void testWritePack2() throws IOException {
278 writeVerifyPack2(false);
279 }
280
281
282
283
284
285
286
287 @Test
288 public void testWritePack2DeltasReuseRefs() throws IOException {
289 writeVerifyPack2(true);
290 }
291
292
293
294
295
296
297
298 @Test
299 public void testWritePack2DeltasReuseOffsets() throws IOException {
300 config.setDeltaBaseAsOffset(true);
301 writeVerifyPack2(true);
302 }
303
304
305
306
307
308
309
310
311 @Test
312 public void testWritePack2DeltasCRC32Copy() throws IOException {
313 final File packDir = db.getObjectDatabase().getPackDirectory();
314 final PackFile crc32Pack = new PackFile(packDir,
315 "pack-34be9032ac282b11fa9babdc2b2a93ca996c9c2f.pack");
316 final PackFile crc32Idx = new PackFile(packDir,
317 "pack-34be9032ac282b11fa9babdc2b2a93ca996c9c2f.idx");
318 copyFile(JGitTestUtil.getTestResourceFile(
319 "pack-34be9032ac282b11fa9babdc2b2a93ca996c9c2f.idxV2"),
320 crc32Idx);
321 db.openPack(crc32Pack);
322
323 writeVerifyPack2(true);
324 }
325
326
327
328
329
330
331
332
333
334 @Test
335 public void testWritePack3() throws MissingObjectException, IOException {
336 config.setReuseDeltas(false);
337 final ObjectId forcedOrder[] = new ObjectId[] {
338 ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"),
339 ObjectId.fromString("c59759f143fb1fe21c197981df75a7ee00290799"),
340 ObjectId.fromString("aabf2ffaec9b497f0950352b3e582d73035c2035"),
341 ObjectId.fromString("902d5476fa249b7abc9d84c611577a81381f0327"),
342 ObjectId.fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3") ,
343 ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259") };
344 try (RevWalk parser = new RevWalk(db)) {
345 final RevObject forcedOrderRevs[] = new RevObject[forcedOrder.length];
346 for (int i = 0; i < forcedOrder.length; i++)
347 forcedOrderRevs[i] = parser.parseAny(forcedOrder[i]);
348
349 createVerifyOpenPack(Arrays.asList(forcedOrderRevs));
350 }
351
352 assertEquals(forcedOrder.length, writer.getObjectCount());
353 verifyObjectsOrder(forcedOrder);
354 assertEquals("ed3f96b8327c7c66b0f8f70056129f0769323d86", writer
355 .computeName().name());
356 }
357
358
359
360
361
362
363
364
365 @Test
366 public void testWritePack4() throws IOException {
367 writeVerifyPack4(false);
368 }
369
370
371
372
373
374
375
376 @Test
377 public void testWritePack4ThinPack() throws IOException {
378 writeVerifyPack4(true);
379 }
380
381
382
383
384
385
386
387
388 @Test
389 public void testWritePack2SizeDeltasVsNoDeltas() throws Exception {
390 config.setReuseDeltas(false);
391 config.setDeltaCompress(false);
392 testWritePack2();
393 final long sizePack2NoDeltas = os.size();
394 tearDown();
395 setUp();
396 testWritePack2DeltasReuseRefs();
397 final long sizePack2DeltasRefs = os.size();
398
399 assertTrue(sizePack2NoDeltas > sizePack2DeltasRefs);
400 }
401
402
403
404
405
406
407
408
409
410 @Test
411 public void testWritePack2SizeOffsetsVsRefs() throws Exception {
412 testWritePack2DeltasReuseRefs();
413 final long sizePack2DeltasRefs = os.size();
414 tearDown();
415 setUp();
416 testWritePack2DeltasReuseOffsets();
417 final long sizePack2DeltasOffsets = os.size();
418
419 assertTrue(sizePack2DeltasRefs > sizePack2DeltasOffsets);
420 }
421
422
423
424
425
426
427
428
429 @Test
430 public void testWritePack4SizeThinVsNoThin() throws Exception {
431 testWritePack4();
432 final long sizePack4 = os.size();
433 tearDown();
434 setUp();
435 testWritePack4ThinPack();
436 final long sizePack4Thin = os.size();
437
438 assertTrue(sizePack4 > sizePack4Thin);
439 }
440
441 @Test
442 public void testDeltaStatistics() throws Exception {
443 config.setDeltaCompress(true);
444
445 FileRepository repo = createBareRepository();
446 ArrayList<RevObject> blobs = new ArrayList<>();
447 try (TestRepository<FileRepository> testRepo = new TestRepository<>(
448 repo)) {
449 blobs.add(testRepo.blob(genDeltableData(1000)));
450 blobs.add(testRepo.blob(genDeltableData(1005)));
451 try (PackWriter pw = new PackWriter(repo)) {
452 NullProgressMonitor m = NullProgressMonitor.INSTANCE;
453 pw.preparePack(blobs.iterator());
454 pw.writePack(m, m, os);
455 PackStatistics stats = pw.getStatistics();
456 assertEquals(1, stats.getTotalDeltas());
457 assertTrue("Delta bytes not set.",
458 stats.byObjectType(OBJ_BLOB).getDeltaBytes() > 0);
459 }
460 }
461 }
462
463
464 private String genDeltableData(int length) {
465 assertTrue("Generated data must have a length > 0", length > 0);
466 char[] data = {'a', 'b', 'c', '\n'};
467 StringBuilder builder = new StringBuilder(length);
468 for (int i = 0; i < length; i++) {
469 builder.append(data[i % 4]);
470 }
471 return builder.toString();
472 }
473
474
475 @Test
476 public void testWriteIndex() throws Exception {
477 config.setIndexVersion(2);
478 writeVerifyPack4(false);
479
480 PackFile packFile = pack.getPackFile();
481 PackFile indexFile = packFile.create(PackExt.INDEX);
482
483
484 final PackIndex idx1 = PackIndex.open(indexFile);
485 assertTrue(idx1 instanceof PackIndexV2);
486 assertEquals(0x4743F1E4L, idx1.findCRC32(ObjectId
487 .fromString("82c6b885ff600be425b4ea96dee75dca255b69e7")));
488
489
490 final File idx2File = new File(indexFile.getAbsolutePath() + ".2");
491 try (FileOutputStream is = new FileOutputStream(idx2File)) {
492 writer.writeIndex(is);
493 }
494 final PackIndex idx2 = PackIndex.open(idx2File);
495 assertTrue(idx2 instanceof PackIndexV2);
496 assertEquals(idx1.getObjectCount(), idx2.getObjectCount());
497 assertEquals(idx1.getOffset64Count(), idx2.getOffset64Count());
498
499 for (int i = 0; i < idx1.getObjectCount(); i++) {
500 final ObjectId id = idx1.getObjectId(i);
501 assertEquals(id, idx2.getObjectId(i));
502 assertEquals(idx1.findOffset(id), idx2.findOffset(id));
503 assertEquals(idx1.findCRC32(id), idx2.findCRC32(id));
504 }
505 }
506
507 @Test
508 public void testExclude() throws Exception {
509
510 FileRepository repo = createBareRepository();
511
512 try (TestRepository<FileRepository> testRepo = new TestRepository<>(
513 repo)) {
514 BranchBuilder bb = testRepo.branch("refs/heads/master");
515 contentA = testRepo.blob("A");
516 c1 = bb.commit().add("f", contentA).create();
517 testRepo.getRevWalk().parseHeaders(c1);
518 PackIndex pf1 = writePack(repo, wants(c1), EMPTY_ID_SET);
519 assertContent(pf1, Arrays.asList(c1.getId(), c1.getTree().getId(),
520 contentA.getId()));
521 contentB = testRepo.blob("B");
522 c2 = bb.commit().add("f", contentB).create();
523 testRepo.getRevWalk().parseHeaders(c2);
524 PackIndex pf2 = writePack(repo, wants(c2),
525 Sets.of((ObjectIdSet) pf1));
526 assertContent(pf2, Arrays.asList(c2.getId(), c2.getTree().getId(),
527 contentB.getId()));
528 }
529 }
530
531 private static void assertContent(PackIndex pi, List<ObjectId> expected) {
532 assertEquals("Pack index has wrong size.", expected.size(),
533 pi.getObjectCount());
534 for (int i = 0; i < pi.getObjectCount(); i++)
535 assertTrue(
536 "Pack index didn't contain the expected id "
537 + pi.getObjectId(i),
538 expected.contains(pi.getObjectId(i)));
539 }
540
541 @Test
542 public void testShallowIsMinimalDepth1() throws Exception {
543 try (FileRepository repo = setupRepoForShallowFetch()) {
544 PackIndex idx = writeShallowPack(repo, 1, wants(c2), NONE, NONE);
545 assertContent(idx, Arrays.asList(c2.getId(), c2.getTree().getId(),
546 contentA.getId(), contentB.getId()));
547
548
549 idx = writeShallowPack(repo, 1, wants(c5), haves(c2), shallows(c2));
550 assertContent(idx, Arrays.asList(c5.getId(), c5.getTree().getId(),
551 contentC.getId(), contentD.getId(), contentE.getId()));
552 }
553 }
554
555 @Test
556 public void testShallowIsMinimalDepth2() throws Exception {
557 try (FileRepository repo = setupRepoForShallowFetch()) {
558 PackIndex idx = writeShallowPack(repo, 2, wants(c2), NONE, NONE);
559 assertContent(idx,
560 Arrays.asList(c1.getId(), c2.getId(), c1.getTree().getId(),
561 c2.getTree().getId(), contentA.getId(),
562 contentB.getId()));
563
564
565 idx = writeShallowPack(repo, 2, wants(c5), haves(c1, c2),
566 shallows(c1));
567 assertContent(idx,
568 Arrays.asList(c4.getId(), c5.getId(), c4.getTree().getId(),
569 c5.getTree().getId(), contentC.getId(),
570 contentD.getId(), contentE.getId()));
571 }
572 }
573
574 @Test
575 public void testShallowFetchShallowParentDepth1() throws Exception {
576 try (FileRepository repo = setupRepoForShallowFetch()) {
577 PackIndex idx = writeShallowPack(repo, 1, wants(c5), NONE, NONE);
578 assertContent(idx, Arrays.asList(c5.getId(), c5.getTree().getId(),
579 contentA.getId(), contentB.getId(), contentC.getId(),
580 contentD.getId(), contentE.getId()));
581
582 idx = writeShallowPack(repo, 1, wants(c4), haves(c5), shallows(c5));
583 assertContent(idx, Arrays.asList(c4.getId(), c4.getTree().getId()));
584 }
585 }
586
587 @Test
588 public void testShallowFetchShallowParentDepth2() throws Exception {
589 try (FileRepository repo = setupRepoForShallowFetch()) {
590 PackIndex idx = writeShallowPack(repo, 2, wants(c5), NONE, NONE);
591 assertContent(idx,
592 Arrays.asList(c4.getId(), c5.getId(), c4.getTree().getId(),
593 c5.getTree().getId(), contentA.getId(),
594 contentB.getId(), contentC.getId(),
595 contentD.getId(), contentE.getId()));
596
597 idx = writeShallowPack(repo, 2, wants(c3), haves(c4, c5),
598 shallows(c4));
599 assertContent(idx, Arrays.asList(c2.getId(), c3.getId(),
600 c2.getTree().getId(), c3.getTree().getId()));
601 }
602 }
603
604 @Test
605 public void testShallowFetchShallowAncestorDepth1() throws Exception {
606 try (FileRepository repo = setupRepoForShallowFetch()) {
607 PackIndex idx = writeShallowPack(repo, 1, wants(c5), NONE, NONE);
608 assertContent(idx, Arrays.asList(c5.getId(), c5.getTree().getId(),
609 contentA.getId(), contentB.getId(), contentC.getId(),
610 contentD.getId(), contentE.getId()));
611
612 idx = writeShallowPack(repo, 1, wants(c3), haves(c5), shallows(c5));
613 assertContent(idx, Arrays.asList(c3.getId(), c3.getTree().getId()));
614 }
615 }
616
617 @Test
618 public void testShallowFetchShallowAncestorDepth2() throws Exception {
619 try (FileRepository repo = setupRepoForShallowFetch()) {
620 PackIndex idx = writeShallowPack(repo, 2, wants(c5), NONE, NONE);
621 assertContent(idx,
622 Arrays.asList(c4.getId(), c5.getId(), c4.getTree().getId(),
623 c5.getTree().getId(), contentA.getId(),
624 contentB.getId(), contentC.getId(),
625 contentD.getId(), contentE.getId()));
626
627 idx = writeShallowPack(repo, 2, wants(c2), haves(c4, c5),
628 shallows(c4));
629 assertContent(idx, Arrays.asList(c1.getId(), c2.getId(),
630 c1.getTree().getId(), c2.getTree().getId()));
631 }
632 }
633
634 @Test
635 public void testTotalPackFilesScanWhenSearchForReuseTimeoutNotSet()
636 throws Exception {
637 FileRepository fileRepository = setUpRepoWithMultiplePackfiles();
638 PackWriter mockedPackWriter = Mockito
639 .spy(new PackWriter(config, fileRepository.newObjectReader()));
640
641 doNothing().when(mockedPackWriter).select(any(), any());
642
643 try (FileOutputStream packOS = new FileOutputStream(
644 getPackFileToWrite(fileRepository, mockedPackWriter))) {
645 mockedPackWriter.writePack(NullProgressMonitor.INSTANCE,
646 NullProgressMonitor.INSTANCE, packOS);
647 }
648
649 long numberOfPackFiles = new GC(fileRepository)
650 .getStatistics().numberOfPackFiles;
651 int expectedSelectCalls =
652
653 2 * (int) numberOfPackFiles +
654
655 1;
656 verify(mockedPackWriter, times(expectedSelectCalls)).select(any(),
657 any());
658 }
659
660 @Test
661 public void testTotalPackFilesScanWhenSkippingSearchForReuseTimeoutCheck()
662 throws Exception {
663 FileRepository fileRepository = setUpRepoWithMultiplePackfiles();
664 PackConfig packConfig = new PackConfig();
665 packConfig.setSearchForReuseTimeout(Duration.ofSeconds(-1));
666 PackWriter mockedPackWriter = Mockito.spy(
667 new PackWriter(packConfig, fileRepository.newObjectReader()));
668
669 doNothing().when(mockedPackWriter).select(any(), any());
670
671 try (FileOutputStream packOS = new FileOutputStream(
672 getPackFileToWrite(fileRepository, mockedPackWriter))) {
673 mockedPackWriter.writePack(NullProgressMonitor.INSTANCE,
674 NullProgressMonitor.INSTANCE, packOS);
675 }
676
677 long numberOfPackFiles = new GC(fileRepository)
678 .getStatistics().numberOfPackFiles;
679 int expectedSelectCalls =
680
681 2 * (int) numberOfPackFiles +
682
683 1;
684 verify(mockedPackWriter, times(expectedSelectCalls)).select(any(),
685 any());
686 }
687
688 @Test
689 public void testPartialPackFilesScanWhenDoingSearchForReuseTimeoutCheck()
690 throws Exception {
691 FileRepository fileRepository = setUpRepoWithMultiplePackfiles();
692 PackConfig packConfig = new PackConfig();
693 packConfig.setSearchForReuseTimeout(Duration.ofSeconds(-1));
694 PackWriter mockedPackWriter = Mockito.spy(
695 new PackWriter(packConfig, fileRepository.newObjectReader()));
696 mockedPackWriter.enableSearchForReuseTimeout();
697
698 doNothing().when(mockedPackWriter).select(any(), any());
699
700 try (FileOutputStream packOS = new FileOutputStream(
701 getPackFileToWrite(fileRepository, mockedPackWriter))) {
702 mockedPackWriter.writePack(NullProgressMonitor.INSTANCE,
703 NullProgressMonitor.INSTANCE, packOS);
704 }
705
706 int expectedSelectCalls = 3;
707 verify(mockedPackWriter, times(expectedSelectCalls)).select(any(),
708 any());
709 }
710
711
712
713
714
715
716
717
718
719
720
721
722
723 private FileRepository setUpRepoWithMultiplePackfiles() throws Exception {
724 FileRepository fileRepository = createWorkRepository();
725 addRepoToClose(fileRepository);
726 try (Git git = new Git(fileRepository)) {
727
728 git.commit().setMessage("First commit").call();
729 GC gc = new GC(fileRepository);
730 gc.setPackExpireAgeMillis(Long.MAX_VALUE);
731 gc.setExpireAgeMillis(Long.MAX_VALUE);
732
733 gc.gc().get();
734
735 git.commit().setMessage("Second commit").call();
736
737 gc.gc().get();
738
739 git.commit().setMessage("Third commit").call();
740 }
741 return fileRepository;
742 }
743
744 private PackFile getPackFileToWrite(FileRepository fileRepository,
745 PackWriter mockedPackWriter) throws IOException {
746 File packdir = fileRepository.getObjectDatabase().getPackDirectory();
747 PackFile packFile = new PackFile(packdir,
748 mockedPackWriter.computeName(), PackExt.PACK);
749
750 Set<ObjectId> all = new HashSet<>();
751 for (Ref r : fileRepository.getRefDatabase().getRefs()) {
752 all.add(r.getObjectId());
753 }
754
755 mockedPackWriter.preparePack(NullProgressMonitor.INSTANCE, all,
756 PackWriter.NONE);
757 return packFile;
758 }
759
760 private FileRepository setupRepoForShallowFetch() throws Exception {
761 FileRepository repo = createBareRepository();
762
763
764 repo.incrementOpen();
765 try (TestRepository<Repository> r = new TestRepository<>(repo)) {
766 BranchBuilder bb = r.branch("refs/heads/master");
767 contentA = r.blob("A");
768 contentB = r.blob("B");
769 contentC = r.blob("C");
770 contentD = r.blob("D");
771 contentE = r.blob("E");
772 c1 = bb.commit().add("a", contentA).create();
773 c2 = bb.commit().add("b", contentB).create();
774 c3 = bb.commit().add("c", contentC).create();
775 c4 = bb.commit().add("d", contentD).create();
776 c5 = bb.commit().add("e", contentE).create();
777 r.getRevWalk().parseHeaders(c5);
778 return repo;
779 }
780 }
781
782 private static PackIndex writePack(FileRepository repo,
783 Set<? extends ObjectId> want, Set<ObjectIdSet> excludeObjects)
784 throws IOException {
785 try (RevWalk walk = new RevWalk(repo)) {
786 return writePack(repo, walk, 0, want, NONE, excludeObjects);
787 }
788 }
789
790 private static PackIndex writeShallowPack(FileRepository repo, int depth,
791 Set<? extends ObjectId> want, Set<? extends ObjectId> have,
792 Set<? extends ObjectId> shallow) throws IOException {
793
794
795 try (DepthWalk.RevWalk walk = new DepthWalk.RevWalk(repo, depth - 1)) {
796 walk.assumeShallow(shallow);
797 return writePack(repo, walk, depth, want, have, EMPTY_ID_SET);
798 }
799 }
800
801 private static PackIndex writePack(FileRepository repo, RevWalk walk,
802 int depth, Set<? extends ObjectId> want,
803 Set<? extends ObjectId> have, Set<ObjectIdSet> excludeObjects)
804 throws IOException {
805 try (PackWriter pw = new PackWriter(repo)) {
806 pw.setDeltaBaseAsOffset(true);
807 pw.setReuseDeltaCommits(false);
808 for (ObjectIdSet idx : excludeObjects) {
809 pw.excludeObjects(idx);
810 }
811 if (depth > 0) {
812 pw.setShallowPack(depth, null);
813 }
814
815 ObjectWalk ow = walk.toObjectWalkWithSameObjects();
816
817 pw.preparePack(NullProgressMonitor.INSTANCE, ow, want, have, NONE);
818 File packdir = repo.getObjectDatabase().getPackDirectory();
819 PackFile packFile = new PackFile(packdir, pw.computeName(),
820 PackExt.PACK);
821 try (FileOutputStream packOS = new FileOutputStream(packFile)) {
822 pw.writePack(NullProgressMonitor.INSTANCE,
823 NullProgressMonitor.INSTANCE, packOS);
824 }
825 PackFile idxFile = packFile.create(PackExt.INDEX);
826 try (FileOutputStream idxOS = new FileOutputStream(idxFile)) {
827 pw.writeIndex(idxOS);
828 }
829 return PackIndex.open(idxFile);
830 }
831 }
832
833
834
835
836 private void writeVerifyPack1() throws IOException {
837 final HashSet<ObjectId> interestings = new HashSet<>();
838 interestings.add(ObjectId
839 .fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"));
840 createVerifyOpenPack(interestings, NONE, false, false);
841
842 final ObjectId expectedOrder[] = new ObjectId[] {
843 ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"),
844 ObjectId.fromString("c59759f143fb1fe21c197981df75a7ee00290799"),
845 ObjectId.fromString("540a36d136cf413e4b064c2b0e0a4db60f77feab"),
846 ObjectId.fromString("aabf2ffaec9b497f0950352b3e582d73035c2035"),
847 ObjectId.fromString("902d5476fa249b7abc9d84c611577a81381f0327"),
848 ObjectId.fromString("4b825dc642cb6eb9a060e54bf8d69288fbee4904"),
849 ObjectId.fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3"),
850 ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259") };
851
852 assertEquals(expectedOrder.length, writer.getObjectCount());
853 verifyObjectsOrder(expectedOrder);
854 assertEquals("34be9032ac282b11fa9babdc2b2a93ca996c9c2f", writer
855 .computeName().name());
856 }
857
858 private void writeVerifyPack2(boolean deltaReuse) throws IOException {
859 config.setReuseDeltas(deltaReuse);
860 final HashSet<ObjectId> interestings = new HashSet<>();
861 interestings.add(ObjectId
862 .fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"));
863 final HashSet<ObjectId> uninterestings = new HashSet<>();
864 uninterestings.add(ObjectId
865 .fromString("540a36d136cf413e4b064c2b0e0a4db60f77feab"));
866 createVerifyOpenPack(interestings, uninterestings, false, false);
867
868 final ObjectId expectedOrder[] = new ObjectId[] {
869 ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"),
870 ObjectId.fromString("c59759f143fb1fe21c197981df75a7ee00290799"),
871 ObjectId.fromString("aabf2ffaec9b497f0950352b3e582d73035c2035"),
872 ObjectId.fromString("902d5476fa249b7abc9d84c611577a81381f0327"),
873 ObjectId.fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3") ,
874 ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259") };
875 if (!config.isReuseDeltas() && !config.isDeltaCompress()) {
876
877 swap(expectedOrder, 4, 5);
878 }
879 assertEquals(expectedOrder.length, writer.getObjectCount());
880 verifyObjectsOrder(expectedOrder);
881 assertEquals("ed3f96b8327c7c66b0f8f70056129f0769323d86", writer
882 .computeName().name());
883 }
884
885 private static void swap(ObjectId[] arr, int a, int b) {
886 ObjectId tmp = arr[a];
887 arr[a] = arr[b];
888 arr[b] = tmp;
889 }
890
891 private void writeVerifyPack4(final boolean thin) throws IOException {
892 final HashSet<ObjectId> interestings = new HashSet<>();
893 interestings.add(ObjectId
894 .fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"));
895 final HashSet<ObjectId> uninterestings = new HashSet<>();
896 uninterestings.add(ObjectId
897 .fromString("c59759f143fb1fe21c197981df75a7ee00290799"));
898 createVerifyOpenPack(interestings, uninterestings, thin, false);
899
900 final ObjectId writtenObjects[] = new ObjectId[] {
901 ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"),
902 ObjectId.fromString("aabf2ffaec9b497f0950352b3e582d73035c2035"),
903 ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259") };
904 assertEquals(writtenObjects.length, writer.getObjectCount());
905 ObjectId expectedObjects[];
906 if (thin) {
907 expectedObjects = new ObjectId[4];
908 System.arraycopy(writtenObjects, 0, expectedObjects, 0,
909 writtenObjects.length);
910 expectedObjects[3] = ObjectId
911 .fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3");
912
913 } else {
914 expectedObjects = writtenObjects;
915 }
916 verifyObjectsOrder(expectedObjects);
917 assertEquals("cded4b74176b4456afa456768b2b5aafb41c44fc", writer
918 .computeName().name());
919 }
920
921 private void createVerifyOpenPack(final Set<ObjectId> interestings,
922 final Set<ObjectId> uninterestings, final boolean thin,
923 final boolean ignoreMissingUninteresting)
924 throws MissingObjectException, IOException {
925 createVerifyOpenPack(interestings, uninterestings, thin,
926 ignoreMissingUninteresting, false);
927 }
928
929 private void createVerifyOpenPack(final Set<ObjectId> interestings,
930 final Set<ObjectId> uninterestings, final boolean thin,
931 final boolean ignoreMissingUninteresting, boolean useBitmaps)
932 throws MissingObjectException, IOException {
933 NullProgressMonitor m = NullProgressMonitor.INSTANCE;
934 writer = new PackWriter(config, db.newObjectReader());
935 writer.setUseBitmaps(useBitmaps);
936 writer.setThin(thin);
937 writer.setIgnoreMissingUninteresting(ignoreMissingUninteresting);
938 writer.preparePack(m, interestings, uninterestings);
939 writer.writePack(m, m, os);
940 writer.close();
941 verifyOpenPack(thin);
942 }
943
944 private void createVerifyOpenPack(List<RevObject> objectSource)
945 throws MissingObjectException, IOException {
946 NullProgressMonitor m = NullProgressMonitor.INSTANCE;
947 writer = new PackWriter(config, db.newObjectReader());
948 writer.preparePack(objectSource.iterator());
949 assertEquals(objectSource.size(), writer.getObjectCount());
950 writer.writePack(m, m, os);
951 writer.close();
952 verifyOpenPack(false);
953 }
954
955 private void verifyOpenPack(boolean thin) throws IOException {
956 final byte[] packData = os.toByteArray();
957
958 if (thin) {
959 PackParser p = index(packData);
960 try {
961 p.parse(NullProgressMonitor.INSTANCE);
962 fail("indexer should grumble about missing object");
963 } catch (IOException x) {
964
965 }
966 }
967
968 ObjectDirectoryPackParser p = (ObjectDirectoryPackParser) index(packData);
969 p.setKeepEmpty(true);
970 p.setAllowThin(thin);
971 p.setIndexVersion(2);
972 p.parse(NullProgressMonitor.INSTANCE);
973 pack = p.getPack();
974 assertNotNull("have PackFile after parsing", pack);
975 }
976
977 private PackParser index(byte[] packData) throws IOException {
978 if (inserter == null)
979 inserter = dst.newObjectInserter();
980 return inserter.newPackParser(new ByteArrayInputStream(packData));
981 }
982
983 private void verifyObjectsOrder(ObjectId objectsOrder[]) {
984 final List<PackIndex.MutableEntry> entries = new ArrayList<>();
985
986 for (MutableEntry me : pack) {
987 entries.add(me.cloneEntry());
988 }
989 Collections.sort(entries, (MutableEntry o1, MutableEntry o2) -> Long
990 .signum(o1.getOffset() - o2.getOffset()));
991
992 int i = 0;
993 for (MutableEntry me : entries) {
994 assertEquals(objectsOrder[i++].toObjectId(), me.toObjectId());
995 }
996 }
997
998 private static Set<ObjectId> haves(ObjectId... objects) {
999 return Sets.of(objects);
1000 }
1001
1002 private static Set<ObjectId> wants(ObjectId... objects) {
1003 return Sets.of(objects);
1004 }
1005
1006 private static Set<ObjectId> shallows(ObjectId... objects) {
1007 return Sets.of(objects);
1008 }
1009 }