Skip to content

Commit 2c233cb

Browse files
hmottestadhmottestad
committed
wip
1 parent 81a33b3 commit 2c233cb

File tree

3 files changed

+43
-10
lines changed

3 files changed

+43
-10
lines changed

‎core/sail/api/src/main/java/org/eclipse/rdf4j/sail/helpers/DirectoryLockManager.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
package org.eclipse.rdf4j.sail.helpers;
1212

1313
import java.io.BufferedReader;
14+
import java.io.BufferedWriter;
1415
import java.io.File;
1516
import java.io.FileReader;
1617
import java.io.FileWriter;

‎core/sail/nativerdf/src/main/java/org/eclipse/rdf4j/sail/nativerdf/NativeStore.java‎

Lines changed: 29 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import org.apache.commons.io.FileUtils;
2525
import org.eclipse.rdf4j.collection.factory.api.CollectionFactory;
2626
import org.eclipse.rdf4j.collection.factory.mapdb.MapDb3CollectionFactory;
27+
import org.eclipse.rdf4j.common.annotation.Experimental;
2728
import org.eclipse.rdf4j.common.annotation.InternalUseOnly;
2829
import org.eclipse.rdf4j.common.concurrent.locks.Lock;
2930
import org.eclipse.rdf4j.common.concurrent.locks.LockManager;
@@ -275,85 +276,105 @@ public void setNamespaceIDCacheSize(int namespaceIDCacheSize) {
275276
this.namespaceIDCacheSize = namespaceIDCacheSize;
276277
}
277278

279+
@Experimental
278280
public void setWalMaxSegmentBytes(long walMaxSegmentBytes) {
279281
this.walMaxSegmentBytes = walMaxSegmentBytes;
280282
}
281283

284+
@Experimental
282285
public long getWalMaxSegmentBytes() {
283286
return walMaxSegmentBytes;
284287
}
285288

289+
@Experimental
286290
public void setWalQueueCapacity(int walQueueCapacity) {
287291
this.walQueueCapacity = walQueueCapacity;
288292
}
289293

294+
@Experimental
290295
public int getWalQueueCapacity() {
291296
return walQueueCapacity;
292297
}
293298

299+
@Experimental
294300
public void setWalBatchBufferBytes(int walBatchBufferBytes) {
295301
this.walBatchBufferBytes = walBatchBufferBytes;
296302
}
297303

304+
@Experimental
298305
public int getWalBatchBufferBytes() {
299306
return walBatchBufferBytes;
300307
}
301308

309+
@Experimental
302310
public void setWalSyncPolicy(ValueStoreWalConfig.SyncPolicy walSyncPolicy) {
303311
this.walSyncPolicy = walSyncPolicy;
304312
}
305313

314+
@Experimental
306315
public ValueStoreWalConfig.SyncPolicy getWalSyncPolicy() {
307316
return walSyncPolicy;
308317
}
309318

319+
@Experimental
310320
public void setWalSyncIntervalMillis(long walSyncIntervalMillis) {
311321
this.walSyncIntervalMillis = walSyncIntervalMillis;
312322
}
313323

324+
@Experimental
314325
public long getWalSyncIntervalMillis() {
315326
return walSyncIntervalMillis;
316327
}
317328

329+
@Experimental
318330
public void setWalIdlePollIntervalMillis(long walIdlePollIntervalMillis) {
319331
this.walIdlePollIntervalMillis = walIdlePollIntervalMillis;
320332
}
321333

334+
@Experimental
322335
public long getWalIdlePollIntervalMillis() {
323336
return walIdlePollIntervalMillis;
324337
}
325338

339+
@Experimental
326340
public void setWalDirectoryName(String walDirectoryName) {
327341
this.walDirectoryName = walDirectoryName;
328342
}
329343

344+
@Experimental
330345
public String getWalDirectoryName() {
331346
return walDirectoryName;
332347
}
333348

334349
/** Ensure WAL bootstrap is synchronous during open (before new values are added). */
350+
@Experimental
335351
public void setWalSyncBootstrapOnOpen(boolean walSyncBootstrapOnOpen) {
336352
this.walSyncBootstrapOnOpen = walSyncBootstrapOnOpen;
337353
}
338354

355+
@Experimental
339356
public boolean isWalSyncBootstrapOnOpen() {
340357
return walSyncBootstrapOnOpen;
341358
}
342359

343360
/** Enable automatic ValueStore recovery from WAL during open. */
361+
@Experimental
344362
public void setWalAutoRecoverOnOpen(boolean walAutoRecoverOnOpen) {
345363
this.walAutoRecoverOnOpen = walAutoRecoverOnOpen;
346364
}
347365

366+
@Experimental
348367
public boolean isWalAutoRecoverOnOpen() {
349368
return walAutoRecoverOnOpen;
350369
}
351370

352371
/** Enable or disable the ValueStore WAL entirely. */
372+
@Experimental
353373
public void setWalEnabled(boolean walEnabled) {
354374
this.walEnabled = walEnabled;
355375
}
356376

377+
@Experimental
357378
public boolean isWalEnabled() {
358379
return walEnabled;
359380
}
@@ -442,8 +463,13 @@ protected void initializeInternal() throws SailException {
442463

443464
try {
444465
Path versionPath = new File(dataDir, "nativerdf.ver").toPath();
445-
String version = versionPath.toFile().exists() ? Files.readString(versionPath, StandardCharsets.UTF_8)
446-
: null;
466+
String version;
467+
try {
468+
version = Files.readString(versionPath, StandardCharsets.UTF_8);
469+
} catch (Exception e) {
470+
version = null;
471+
}
472+
447473
if (!VERSION.equals(version) && upgradeStore(dataDir, version)) {
448474
logger.debug("Data store upgraded to version " + VERSION);
449475
Files.writeString(versionPath, VERSION, StandardCharsets.UTF_8,
@@ -467,7 +493,7 @@ protected void initializeInternal() throws SailException {
467493
walSyncBootstrapOnOpen,
468494
walAutoRecoverOnOpen,
469495
walEnabled);
470-
this.store = new SnapshotSailStore(mainStore, () -> new MemoryOverflowIntoNativeStore()) {
496+
this.store = new SnapshotSailStore(mainStore, MemoryOverflowIntoNativeStore::new) {
471497

472498
@Override
473499
public SailSource getExplicitSailSource() {

‎core/sail/nativerdf/src/test/java/org/eclipse/rdf4j/sail/nativerdf/NativeSailStoreCorruptionTestIT.java‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -357,6 +357,8 @@ public void testCorruptValuesContextsFile() throws IOException {
357357
public void testCorruptValuesPoscAllocFile() throws IOException {
358358
String expected = getStatements().stream().map(Object::toString).reduce((a, b) -> a + "\n" + b).get();
359359
repo.shutDown();
360+
((NativeStore) repo.getSail()).setWalEnabled(false);
361+
360362
String file = "triples-posc.alloc";
361363
File nativeStoreFile = new File(dataDir, file);
362364
long fileSize = nativeStoreFile.length();
@@ -365,6 +367,7 @@ public void testCorruptValuesPoscAllocFile() throws IOException {
365367
restoreFile(dataDir, file);
366368
overwriteByteInFile(nativeStoreFile, i, 0x0);
367369
repo.init();
370+
368371
List<Statement> list = getStatements();
369372
assertEquals(6, list.size(), "Failed at byte position " + i);
370373
String actual = list.stream().map(Object::toString).reduce((a, b) -> a + "\n" + b).get();
@@ -379,8 +382,7 @@ public void testCorruptValuesPoscDataFile() throws IOException {
379382
String expected = getStatements().stream().map(Object::toString).reduce((a, b) -> a + "\n" + b).get();
380383
repo.shutDown();
381384

382-
NativeStore sail = (NativeStore) repo.getSail();
383-
sail.setWalEnabled(false);
385+
((NativeStore) repo.getSail()).setWalEnabled(false);
384386

385387
String file = "triples-posc.dat";
386388
File nativeStoreFile = new File(dataDir, file);
@@ -393,11 +395,9 @@ public void testCorruptValuesPoscDataFile() throws IOException {
393395
NativeStore.SOFT_FAIL_ON_CORRUPT_DATA_AND_REPAIR_INDEXES = true;
394396
restoreFile(dataDir, file);
395397
overwriteByteInFile(nativeStoreFile, i, 0x0);
396-
try {
397-
repo.init();
398-
} catch (Exception e) {
399-
System.out.println();
400-
}
398+
399+
repo.init();
400+
401401
List<Statement> list = getStatements();
402402
assertEquals(6, list.size(), "Failed at byte position " + i);
403403
String actual = list.stream().map(Object::toString).reduce((a, b) -> a + "\n" + b).get();
@@ -411,11 +411,16 @@ public void testCorruptValuesPoscDataFile() throws IOException {
411411
public void testCorruptValuesSpocAllocFile() throws IOException {
412412
String expected = getStatements().stream().map(Object::toString).reduce((a, b) -> a + "\n" + b).get();
413413
repo.shutDown();
414+
((NativeStore) repo.getSail()).setWalEnabled(false);
415+
414416
String file = "triples-spoc.alloc";
415417
File nativeStoreFile = new File(dataDir, file);
416418
long fileSize = nativeStoreFile.length();
417419

418420
for (long i = 4; i < fileSize; i++) {
421+
if (i % 1024 == 0) {
422+
System.out.println("Testing byte " + i);
423+
}
419424
restoreFile(dataDir, file);
420425
overwriteByteInFile(nativeStoreFile, i, 0x0);
421426
repo.init();
@@ -432,6 +437,7 @@ public void testCorruptValuesSpocAllocFile() throws IOException {
432437
public void testCorruptValuesSpocDataFile() throws IOException {
433438
String expected = getStatements().stream().map(Object::toString).reduce((a, b) -> a + "\n" + b).get();
434439
repo.shutDown();
440+
((NativeStore) repo.getSail()).setWalEnabled(false);
435441

436442
NativeStore sail = (NativeStore) repo.getSail();
437443
sail.setWalEnabled(false);

0 commit comments

Comments
 (0)