View Javadoc
1   /*
2    * Copyright (C) 2012, Christian Halstrick <christian.halstrick@sap.com> and others
3    *
4    * This program and the accompanying materials are made available under the
5    * terms of the Eclipse Distribution License v. 1.0 which is available at
6    * https://www.eclipse.org/org/documents/edl-v10.php.
7    *
8    * SPDX-License-Identifier: BSD-3-Clause
9    */
10  
11  package org.eclipse.jgit.internal.storage.file;
12  
13  import static org.junit.Assert.assertEquals;
14  import static org.junit.Assert.assertFalse;
15  import static org.junit.Assert.assertTrue;
16  
17  import java.io.File;
18  import java.io.IOException;
19  import java.util.ArrayList;
20  import java.util.Collection;
21  import java.util.Date;
22  import java.util.List;
23  
24  import org.eclipse.jgit.junit.TestRepository.BranchBuilder;
25  import org.eclipse.jgit.lib.ConfigConstants;
26  import org.eclipse.jgit.lib.ObjectId;
27  import org.eclipse.jgit.lib.RefUpdate;
28  import org.eclipse.jgit.revwalk.RevCommit;
29  import org.eclipse.jgit.storage.file.FileBasedConfig;
30  import org.eclipse.jgit.storage.pack.PackConfig;
31  import org.junit.Test;
32  import org.junit.experimental.theories.DataPoints;
33  import org.junit.experimental.theories.Theories;
34  import org.junit.experimental.theories.Theory;
35  import org.junit.runner.RunWith;
36  
37  @RunWith(Theories.class)
38  public class GcBasicPackingTest extends GcTestCase {
39  	@DataPoints
40  	public static boolean[] aggressiveValues = { true, false };
41  
42  	@Theory
43  	public void repackEmptyRepo_noPackCreated(boolean aggressive)
44  			throws IOException {
45  		configureGc(gc, aggressive);
46  		gc.repack();
47  		assertEquals(0, repo.getObjectDatabase().getPacks().size());
48  	}
49  
50  	@Theory
51  	public void testPackRepoWithNoRefs(boolean aggressive) throws Exception {
52  		tr.commit().add("A", "A").add("B", "B").create();
53  		stats = gc.getStatistics();
54  		assertEquals(4, stats.numberOfLooseObjects);
55  		assertEquals(0, stats.numberOfPackedObjects);
56  		configureGc(gc, aggressive);
57  		gc.gc().get();
58  		stats = gc.getStatistics();
59  		assertEquals(4, stats.numberOfLooseObjects);
60  		assertEquals(0, stats.numberOfPackedObjects);
61  		assertEquals(0, stats.numberOfPackFiles);
62  		assertEquals(0, stats.numberOfBitmaps);
63  	}
64  
65  	@Theory
66  	public void testPack2Commits(boolean aggressive) throws Exception {
67  		BranchBuilder bb = tr.branch("refs/heads/master");
68  		bb.commit().add("A", "A").add("B", "B").create();
69  		bb.commit().add("A", "A2").add("B", "B2").create();
70  
71  		stats = gc.getStatistics();
72  		assertEquals(8, stats.numberOfLooseObjects);
73  		assertEquals(0, stats.numberOfPackedObjects);
74  		configureGc(gc, aggressive);
75  		gc.gc().get();
76  		stats = gc.getStatistics();
77  		assertEquals(0, stats.numberOfLooseObjects);
78  		assertEquals(8, stats.numberOfPackedObjects);
79  		assertEquals(1, stats.numberOfPackFiles);
80  		assertEquals(2, stats.numberOfBitmaps);
81  	}
82  
83  	@Theory
84  	public void testPack2Commits_noPackFolder(boolean aggressive) throws Exception {
85  		File packDir = repo.getObjectDatabase().getPackDirectory();
86  		assertTrue(packDir.delete());
87  
88  		BranchBuilder bb = tr.branch("refs/heads/master");
89  		bb.commit().add("A", "A").add("B", "B").create();
90  		bb.commit().add("A", "A2").add("B", "B2").create();
91  
92  		stats = gc.getStatistics();
93  		assertEquals(8, stats.numberOfLooseObjects);
94  		assertEquals(0, stats.numberOfPackedObjects);
95  		configureGc(gc, aggressive);
96  		gc.gc().get();
97  		stats = gc.getStatistics();
98  		assertEquals(0, stats.numberOfLooseObjects);
99  		assertEquals(8, stats.numberOfPackedObjects);
100 		assertEquals(1, stats.numberOfPackFiles);
101 		assertEquals(2, stats.numberOfBitmaps);
102 
103 		assertTrue(packDir.exists());
104 	}
105 
106 	@Theory
107 	public void testPackAllObjectsInOnePack(boolean aggressive)
108 			throws Exception {
109 		tr.branch("refs/heads/master").commit().add("A", "A").add("B", "B")
110 				.create();
111 		stats = gc.getStatistics();
112 		assertEquals(4, stats.numberOfLooseObjects);
113 		assertEquals(0, stats.numberOfPackedObjects);
114 		configureGc(gc, aggressive);
115 		gc.gc().get();
116 		stats = gc.getStatistics();
117 		assertEquals(0, stats.numberOfLooseObjects);
118 		assertEquals(4, stats.numberOfPackedObjects);
119 		assertEquals(1, stats.numberOfPackFiles);
120 		assertEquals(1, stats.numberOfBitmaps);
121 
122 		// Do the gc again and check that it hasn't changed anything
123 		gc.gc().get();
124 		stats = gc.getStatistics();
125 		assertEquals(0, stats.numberOfLooseObjects);
126 		assertEquals(4, stats.numberOfPackedObjects);
127 		assertEquals(1, stats.numberOfPackFiles);
128 		assertEquals(1, stats.numberOfBitmaps);
129 	}
130 
131 	@Theory
132 	public void testPackCommitsAndLooseOne(boolean aggressive)
133 			throws Exception {
134 		BranchBuilder bb = tr.branch("refs/heads/master");
135 		RevCommit first = bb.commit().add("A", "A").add("B", "B").create();
136 		bb.commit().add("A", "A2").add("B", "B2").create();
137 		tr.update("refs/heads/master", first);
138 
139 		stats = gc.getStatistics();
140 		assertEquals(8, stats.numberOfLooseObjects);
141 		assertEquals(0, stats.numberOfPackedObjects);
142 		configureGc(gc, aggressive);
143 		gc.gc().get();
144 		stats = gc.getStatistics();
145 		assertEquals(0, stats.numberOfLooseObjects);
146 		assertEquals(8, stats.numberOfPackedObjects);
147 		assertEquals(2, stats.numberOfPackFiles);
148 		assertEquals(1, stats.numberOfBitmaps);
149 	}
150 
151 	@Theory
152 	public void testNotPackTwice(boolean aggressive) throws Exception {
153 		BranchBuilder bb = tr.branch("refs/heads/master");
154 		RevCommit first = bb.commit().message("M").add("M", "M").create();
155 		bb.commit().message("B").add("B", "Q").create();
156 		bb.commit().message("A").add("A", "A").create();
157 		RevCommit second = tr.commit().parent(first).message("R").add("R", "Q")
158 				.create();
159 		tr.update("refs/tags/t1", second);
160 
161 		Collection<Pack> oldPacks = tr.getRepository().getObjectDatabase()
162 				.getPacks();
163 		assertEquals(0, oldPacks.size());
164 		stats = gc.getStatistics();
165 		assertEquals(11, stats.numberOfLooseObjects);
166 		assertEquals(0, stats.numberOfPackedObjects);
167 
168 		gc.setExpireAgeMillis(0);
169 		fsTick();
170 		configureGc(gc, aggressive);
171 		gc.gc().get();
172 		stats = gc.getStatistics();
173 		assertEquals(0, stats.numberOfLooseObjects);
174 
175 		List<Pack> packs = new ArrayList<>(
176 				repo.getObjectDatabase().getPacks());
177 		assertEquals(11, packs.get(0).getObjectCount());
178 	}
179 
180 	@Test
181 	public void testDonePruneTooYoungPacks() throws Exception {
182 		BranchBuilder bb = tr.branch("refs/heads/master");
183 		bb.commit().message("M").add("M", "M").create();
184 
185 		String tempRef = "refs/heads/soon-to-be-unreferenced";
186 		BranchBuilder bb2 = tr.branch(tempRef);
187 		bb2.commit().message("M").add("M", "M").create();
188 
189 		gc.setExpireAgeMillis(0);
190 		gc.gc().get();
191 		stats = gc.getStatistics();
192 		assertEquals(0, stats.numberOfLooseObjects);
193 		assertEquals(4, stats.numberOfPackedObjects);
194 		assertEquals(1, stats.numberOfPackFiles);
195 		File oldPackfile = tr.getRepository().getObjectDatabase().getPacks()
196 				.iterator().next().getPackFile();
197 
198 		fsTick();
199 
200 		// delete the temp ref, orphaning its commit
201 		RefUpdate update = tr.getRepository().getRefDatabase().newUpdate(tempRef, false);
202 		update.setForceUpdate(true);
203 		update.delete();
204 
205 		bb.commit().message("B").add("B", "Q").create();
206 
207 		// The old packfile is too young to be deleted. We should end up with
208 		// two pack files
209 		gc.setExpire(new Date(oldPackfile.lastModified() - 1));
210 		gc.gc().get();
211 		stats = gc.getStatistics();
212 		assertEquals(0, stats.numberOfLooseObjects);
213 		// if objects exist in multiple packFiles then they are counted multiple
214 		// times
215 		assertEquals(10, stats.numberOfPackedObjects);
216 		assertEquals(2, stats.numberOfPackFiles);
217 
218 		// repack again but now without a grace period for loose objects. Since
219 		// we don't have loose objects anymore this shouldn't change anything
220 		gc.setExpireAgeMillis(0);
221 		gc.gc().get();
222 		stats = gc.getStatistics();
223 		assertEquals(0, stats.numberOfLooseObjects);
224 		// if objects exist in multiple packFiles then they are counted multiple
225 		// times
226 		assertEquals(10, stats.numberOfPackedObjects);
227 		assertEquals(2, stats.numberOfPackFiles);
228 
229 		// repack again but now without a grace period for packfiles. We should
230 		// end up with one packfile
231 		gc.setPackExpireAgeMillis(0);
232 
233 		// we want to keep newly-loosened objects though
234 		gc.setExpireAgeMillis(-1);
235 
236 		gc.gc().get();
237 		stats = gc.getStatistics();
238 		assertEquals(1, stats.numberOfLooseObjects);
239 		// if objects exist in multiple packFiles then they are counted multiple
240 		// times
241 		assertEquals(6, stats.numberOfPackedObjects);
242 		assertEquals(1, stats.numberOfPackFiles);
243 	}
244 
245 	@Test
246 	public void testImmediatePruning() throws Exception {
247 		BranchBuilder bb = tr.branch("refs/heads/master");
248 		bb.commit().message("M").add("M", "M").create();
249 
250 		String tempRef = "refs/heads/soon-to-be-unreferenced";
251 		BranchBuilder bb2 = tr.branch(tempRef);
252 		bb2.commit().message("M").add("M", "M").create();
253 
254 		gc.setExpireAgeMillis(0);
255 		gc.gc().get();
256 		stats = gc.getStatistics();
257 
258 		fsTick();
259 
260 		// delete the temp ref, orphaning its commit
261 		RefUpdate update = tr.getRepository().getRefDatabase().newUpdate(tempRef, false);
262 		update.setForceUpdate(true);
263 		update.delete();
264 
265 		bb.commit().message("B").add("B", "Q").create();
266 
267 		// We want to immediately prune deleted objects
268 		FileBasedConfig config = repo.getConfig();
269 		config.setString(ConfigConstants.CONFIG_GC_SECTION, null,
270 			ConfigConstants.CONFIG_KEY_PRUNEEXPIRE, "now");
271 		config.save();
272 
273 		//And we don't want to keep packs full of dead objects
274 		gc.setPackExpireAgeMillis(0);
275 
276 		gc.gc().get();
277 		stats = gc.getStatistics();
278 		assertEquals(0, stats.numberOfLooseObjects);
279 		assertEquals(6, stats.numberOfPackedObjects);
280 		assertEquals(1, stats.numberOfPackFiles);
281 	}
282 
283 	@Test
284 	public void testPreserveAndPruneOldPacks() throws Exception {
285 		testPreserveOldPacks();
286 		configureGc(gc, false).setPrunePreserved(true);
287 		gc.gc().get();
288 
289 		assertFalse(repo.getObjectDatabase().getPreservedDirectory().exists());
290 	}
291 
292 	private void testPreserveOldPacks() throws Exception {
293 		BranchBuilder bb = tr.branch("refs/heads/master");
294 		bb.commit().message("P").add("P", "P").create();
295 
296 		// pack loose object into packfile
297 		gc.setExpireAgeMillis(0);
298 		gc.gc().get();
299 		PackFile oldPackfile = tr.getRepository().getObjectDatabase().getPacks()
300 				.iterator().next().getPackFile();
301 		assertTrue(oldPackfile.exists());
302 
303 		fsTick();
304 		bb.commit().message("B").add("B", "Q").create();
305 
306 		// repack again but now without a grace period for packfiles. We should
307 		// end up with a new packfile and the old one should be placed in the
308 		// preserved directory
309 		gc.setPackExpireAgeMillis(0);
310 		configureGc(gc, false).setPreserveOldPacks(true);
311 		gc.gc().get();
312 
313 		File preservedPackFile = oldPackfile.createPreservedForDirectory(
314 				repo.getObjectDatabase().getPreservedDirectory());
315 		assertTrue(preservedPackFile.exists());
316 	}
317 
318 	@Test
319 	public void testPruneAndRestoreOldPacks() throws Exception {
320 		String tempRef = "refs/heads/soon-to-be-unreferenced";
321 		BranchBuilder bb = tr.branch(tempRef);
322 		bb.commit().add("A", "A").add("B", "B").create();
323 
324 		// Verify setup conditions
325 		stats = gc.getStatistics();
326 		assertEquals(4, stats.numberOfLooseObjects);
327 		assertEquals(0, stats.numberOfPackedObjects);
328 
329 		// Force all referenced objects into packs (to avoid having loose objects)
330 		configureGc(gc, false);
331 		gc.setExpireAgeMillis(0);
332 		gc.setPackExpireAgeMillis(0);
333 		gc.gc().get();
334 		stats = gc.getStatistics();
335 		assertEquals(0, stats.numberOfLooseObjects);
336 		assertEquals(4, stats.numberOfPackedObjects);
337 		assertEquals(1, stats.numberOfPackFiles);
338 
339 		// Delete the temp ref, orphaning its commit
340 		RefUpdate update = tr.getRepository().getRefDatabase().newUpdate(tempRef, false);
341 		update.setForceUpdate(true);
342 		ObjectId objectId = update.getOldObjectId(); // remember it so we can restore it!
343 		RefUpdate.Result result = update.delete();
344 		assertEquals(RefUpdate.Result.FORCED, result);
345 
346 		fsTick();
347 
348 		// Repack with only orphaned commit, so packfile will be pruned
349 		configureGc(gc, false).setPreserveOldPacks(true);
350 		gc.gc().get();
351 		stats = gc.getStatistics();
352 		assertEquals(0, stats.numberOfLooseObjects);
353 		assertEquals(0, stats.numberOfPackedObjects);
354 		assertEquals(0, stats.numberOfPackFiles);
355 
356 		// Restore the temp ref to the deleted commit, should restore old-packs!
357 		update = tr.getRepository().getRefDatabase().newUpdate(tempRef, false);
358 		update.setNewObjectId(objectId);
359 		update.setExpectedOldObjectId(null);
360 		result = update.update();
361 		assertEquals(RefUpdate.Result.NEW, result);
362 
363 		stats = gc.getStatistics();
364 		assertEquals(4, stats.numberOfPackedObjects);
365 		assertEquals(1, stats.numberOfPackFiles);
366 	}
367 
368 	private PackConfig configureGc(GC myGc, boolean aggressive) {
369 		PackConfig pconfig = new PackConfig(repo);
370 		if (aggressive) {
371 			pconfig.setDeltaSearchWindowSize(250);
372 			pconfig.setMaxDeltaDepth(250);
373 			pconfig.setReuseObjects(false);
374 		} else
375 			pconfig = new PackConfig(repo);
376 		myGc.setPackConfig(pconfig);
377 		return pconfig;
378 	}
379 }