fix unit test on windows

lizhimins · lizhimins · commit 9e6a94f0e5c3 · 2023-11-20T21:07:20.000+08:00
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/index/IndexStoreFile.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/index/IndexStoreFile.java
@@ -457,9 +457,11 @@ public void shutdown() {
             this.fileStatus.set(IndexStatusEnum.SHUTDOWN);
             if (this.mappedFile != null) {
                 this.mappedFile.shutdown(TimeUnit.SECONDS.toMillis(10));
+                this.mappedFile = null;
             }
             if (this.compactMappedFile != null) {
                 this.compactMappedFile.shutdown(TimeUnit.SECONDS.toMillis(10));
+                this.compactMappedFile = null;
             }
         } catch (Exception e) {
             log.error("IndexStoreFile shutdown failed, timestamp: {}, status: {}", this.getTimestamp(), fileStatus.get(), e);
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/index/IndexStoreService.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/index/IndexStoreService.java
@@ -87,8 +87,12 @@ private void doConvertOldFormatFile(String filePath) {
             }
             MappedFile mappedFile = new DefaultMappedFile(file.getPath(), (int) file.length());
             long timestamp = mappedFile.getMappedByteBuffer().getLong(IndexStoreFile.INDEX_BEGIN_TIME_STAMP);
-            mappedFile.renameTo(String.valueOf(new File(file.getParent(), String.valueOf(timestamp))));
-            mappedFile.shutdown(TimeUnit.SECONDS.toMillis(10));
+            if (timestamp <= 0) {
+                mappedFile.destroy(TimeUnit.SECONDS.toMillis(10));
+            } else {
+                mappedFile.renameTo(String.valueOf(new File(file.getParent(), String.valueOf(timestamp))));
+                mappedFile.shutdown(TimeUnit.SECONDS.toMillis(10));
+            }
         } catch (Exception e) {
             log.error("IndexStoreService do convert old format error, file: {}", filePath, e);
         }
@@ -141,7 +145,7 @@ private void recover() {
             log.info("IndexStoreService recover load remote file, timestamp: {}", indexFile.getTimestamp());
         }
 
-        log.info("IndexStoreService recover finished, entrySize: {}, cost: {}ms, filePath: {}",
+        log.info("IndexStoreService recover finished, entrySize: {}, cost: {}ms, directory: {}",
             timeStoreTable.size(), stopwatch.elapsed(TimeUnit.MILLISECONDS), dir.getAbsolutePath());
     }
 
@@ -352,6 +356,7 @@ public void shutdown() {
         for (Map.Entry<Long /* timestamp */, IndexFile> entry : timeStoreTable.entrySet()) {
             entry.getValue().shutdown();
         }
+        this.timeStoreTable.clear();
         log.info("IndexStoreService shutdown gracefully");
     }
 }
diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/posix/PosixFileSegment.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/posix/PosixFileSegment.java
@@ -74,7 +74,7 @@ public PosixFileSegment(TieredMessageStoreConfig storeConfig,
         String clusterBasePath = TieredStoreUtil.getHash(brokerClusterName) + UNDERLINE + brokerClusterName;
         this.fullPath = Paths.get(basePath, clusterBasePath, filePath,
             fileType.toString(), TieredStoreUtil.offset2FileName(baseOffset)).toString();
-        logger.debug("Constructing Posix FileSegment, filePath: {}", fullPath);
+        logger.info("Constructing Posix FileSegment, filePath: {}", fullPath);
 
         createFile();
     }
diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/index/IndexStoreFileTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/index/IndexStoreFileTest.java
@@ -50,16 +50,18 @@ public class IndexStoreFileTest {
     private static final String KEY = "MessageKey";
     private static final Set<String> KEY_SET = Collections.singleton(KEY);
 
+    private String filePath;
     private TieredMessageStoreConfig storeConfig;
     private IndexStoreFile indexStoreFile;
 
     @Before
     public void init() throws IOException {
         TieredStoreExecutor.init();
-        String filePath = Paths.get(System.getProperty("user.home"), "store_test",
-            UUID.randomUUID().toString().replace("-", "").substring(0, 8)).toString();
+        filePath = UUID.randomUUID().toString().replace("-", "").substring(0, 8);
+        String directory = Paths.get(System.getProperty("user.home"), "store_test", filePath).toString();
         storeConfig = new TieredMessageStoreConfig();
-        storeConfig.setTieredStoreFilePath(filePath);
+        storeConfig.setStorePathRootDir(directory);
+        storeConfig.setTieredStoreFilePath(directory);
         storeConfig.setTieredStoreIndexFileMaxHashSlotNum(5);
         storeConfig.setTieredStoreIndexFileMaxIndexNum(20);
         storeConfig.setTieredBackendServiceProvider("org.apache.rocketmq.tieredstore.provider.posix.PosixFileSegment");
@@ -73,6 +75,7 @@ public void shutdown() {
             this.indexStoreFile.destroy();
         }
         TieredStoreTestUtil.destroyMetadataStore();
+        TieredStoreTestUtil.destroyTempDir(storeConfig.getStorePathRootDir());
         TieredStoreTestUtil.destroyTempDir(storeConfig.getTieredStoreFilePath());
         TieredStoreExecutor.shutdown();
     }
@@ -214,15 +217,14 @@ public void recoverFileTest() throws IOException {
     @Test
     public void doCompactionTest() throws Exception {
         long timestamp = indexStoreFile.getTimestamp();
-        indexStoreFile = new IndexStoreFile(storeConfig, System.currentTimeMillis());
         for (int i = 0; i < 10; i++) {
             Assert.assertEquals(AppendResult.SUCCESS, indexStoreFile.putKey(
                 TOPIC_NAME, TOPIC_ID, QUEUE_ID, KEY_SET, MESSAGE_OFFSET, MESSAGE_SIZE, timestamp));
         }
 
         ByteBuffer byteBuffer = indexStoreFile.doCompaction();
         TieredFileSegment fileSegment = new PosixFileSegment(
-            storeConfig, FileSegmentType.INDEX, "store_index", 0L);
+            storeConfig, FileSegmentType.INDEX, filePath, 0L);
         fileSegment.append(byteBuffer, timestamp);
         fileSegment.commit();
         Assert.assertEquals(byteBuffer.limit(), fileSegment.getSize());
@@ -255,7 +257,7 @@ public void queryAsyncFromSegmentFileTest() throws ExecutionException, Interrupt
 
         ByteBuffer byteBuffer = indexStoreFile.doCompaction();
         TieredFileSegment fileSegment = new PosixFileSegment(
-            storeConfig, FileSegmentType.INDEX, "store_index", 0L);
+            storeConfig, FileSegmentType.INDEX, filePath, 0L);
         fileSegment.append(byteBuffer, timestamp);
         fileSegment.commit();
         Assert.assertEquals(byteBuffer.limit(), fileSegment.getSize());
diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/index/IndexStoreServiceTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/index/IndexStoreServiceTest.java
@@ -36,6 +36,8 @@
 import org.apache.rocketmq.common.ThreadFactoryImpl;
 import org.apache.rocketmq.logging.org.slf4j.Logger;
 import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
+import org.apache.rocketmq.store.logfile.DefaultMappedFile;
+import org.apache.rocketmq.tieredstore.TieredStoreTestUtil;
 import org.apache.rocketmq.tieredstore.common.AppendResult;
 import org.apache.rocketmq.tieredstore.common.TieredMessageStoreConfig;
 import org.apache.rocketmq.tieredstore.common.TieredStoreExecutor;
@@ -67,10 +69,11 @@ public class IndexStoreServiceTest {
     @Before
     public void init() throws IOException, ClassNotFoundException, NoSuchMethodException {
         TieredStoreExecutor.init();
-        filePath = Paths.get(System.getProperty("user.home"), "store_test",
-            UUID.randomUUID().toString().replace("-", "").substring(0, 8)).toString();
+        filePath = UUID.randomUUID().toString().replace("-", "").substring(0, 8);
+        String directory = Paths.get(System.getProperty("user.home"), "store_test", filePath).toString();
         storeConfig = new TieredMessageStoreConfig();
-        storeConfig.setStorePathRootDir(filePath);
+        storeConfig.setStorePathRootDir(directory);
+        storeConfig.setTieredStoreFilePath(directory);
         storeConfig.setTieredStoreIndexFileMaxHashSlotNum(5);
         storeConfig.setTieredStoreIndexFileMaxIndexNum(20);
         storeConfig.setTieredBackendServiceProvider("org.apache.rocketmq.tieredstore.provider.posix.PosixFileSegment");
@@ -83,6 +86,9 @@ public void shutdown() {
             indexService.shutdown();
             indexService.destroy();
         }
+        TieredStoreTestUtil.destroyMetadataStore();
+        TieredStoreTestUtil.destroyTempDir(storeConfig.getStorePathRootDir());
+        TieredStoreTestUtil.destroyTempDir(storeConfig.getTieredStoreFilePath());
         TieredStoreExecutor.shutdown();
     }
 
@@ -99,26 +105,23 @@ public void basicServiceTest() throws InterruptedException {
     }
 
     @Test
-    public void doConvertOldFormatTest() {
+    public void doConvertOldFormatTest() throws IOException {
         indexService = new IndexStoreService(fileAllocator, filePath);
         long timestamp = indexService.getTimeStoreTable().firstKey();
         Assert.assertEquals(AppendResult.SUCCESS, indexService.putKey(
             TOPIC_NAME, TOPIC_ID, QUEUE_ID, KEY_SET, MESSAGE_OFFSET, MESSAGE_SIZE, timestamp));
         indexService.shutdown();
 
         File file = new File(Paths.get(filePath, IndexStoreService.FILE_DIRECTORY_NAME, String.valueOf(timestamp)).toString());
-        log.info(filePath);
-        log.info(file.getPath());
-        Assert.assertTrue(file.exists());
-
-        Assert.assertTrue(file.renameTo(new File(file.getParent(), "0000")));
-        file = new File(Paths.get(file.getParent(), "0000").toString());
-        Assert.assertTrue(file.exists());
+        DefaultMappedFile mappedFile = new DefaultMappedFile(file.getName(), (int) file.length());
+        mappedFile.renameTo(String.valueOf(new File(file.getParent(), "0000")));
+        mappedFile.shutdown(10 * 1000);
 
         indexService = new IndexStoreService(fileAllocator, filePath);
         ConcurrentSkipListMap<Long, IndexFile> timeStoreTable = indexService.getTimeStoreTable();
         Assert.assertEquals(1, timeStoreTable.size());
         Assert.assertEquals(Long.valueOf(timestamp), timeStoreTable.firstKey());
+        mappedFile.destroy(10 * 1000);
     }
 
     @Test

Original file line number	Diff line number	Diff line change
`@@ -457,9 +457,11 @@ public void shutdown() {`
`457`	`457`	`this.fileStatus.set(IndexStatusEnum.SHUTDOWN);`
`458`	`458`	`if (this.mappedFile != null) {`
`459`	`459`	`this.mappedFile.shutdown(TimeUnit.SECONDS.toMillis(10));`
	`460`	`+ this.mappedFile = null;`
`460`	`461`	`}`
`461`	`462`	`if (this.compactMappedFile != null) {`
`462`	`463`	`this.compactMappedFile.shutdown(TimeUnit.SECONDS.toMillis(10));`
	`464`	`+ this.compactMappedFile = null;`
`463`	`465`	`}`
`464`	`466`	`} catch (Exception e) {`
`465`	`467`	`log.error("IndexStoreFile shutdown failed, timestamp: {}, status: {}", this.getTimestamp(), fileStatus.get(), e);`
Original file line number	Diff line number	Diff line change
`@@ -87,8 +87,12 @@ private void doConvertOldFormatFile(String filePath) {`
`87`	`87`	`}`
`88`	`88`	`MappedFile mappedFile = new DefaultMappedFile(file.getPath(), (int) file.length());`
`89`	`89`	`long timestamp = mappedFile.getMappedByteBuffer().getLong(IndexStoreFile.INDEX_BEGIN_TIME_STAMP);`
`90`		`- mappedFile.renameTo(String.valueOf(new File(file.getParent(), String.valueOf(timestamp))));`
`91`		`- mappedFile.shutdown(TimeUnit.SECONDS.toMillis(10));`
	`90`	`+ if (timestamp <= 0) {`
	`91`	`+ mappedFile.destroy(TimeUnit.SECONDS.toMillis(10));`
	`92`	`+ } else {`
	`93`	`+ mappedFile.renameTo(String.valueOf(new File(file.getParent(), String.valueOf(timestamp))));`
	`94`	`+ mappedFile.shutdown(TimeUnit.SECONDS.toMillis(10));`
	`95`	`+ }`
`92`	`96`	`} catch (Exception e) {`
`93`	`97`	`log.error("IndexStoreService do convert old format error, file: {}", filePath, e);`
`94`	`98`	`}`
`@@ -141,7 +145,7 @@ private void recover() {`
`141`	`145`	`log.info("IndexStoreService recover load remote file, timestamp: {}", indexFile.getTimestamp());`
`142`	`146`	`}`
`143`	`147`
`144`		`- log.info("IndexStoreService recover finished, entrySize: {}, cost: {}ms, filePath: {}",`
	`148`	`+ log.info("IndexStoreService recover finished, entrySize: {}, cost: {}ms, directory: {}",`
`145`	`149`	`timeStoreTable.size(), stopwatch.elapsed(TimeUnit.MILLISECONDS), dir.getAbsolutePath());`
`146`	`150`	`}`
`147`	`151`
`@@ -352,6 +356,7 @@ public void shutdown() {`
`352`	`356`	`for (Map.Entry<Long /* timestamp */, IndexFile> entry : timeStoreTable.entrySet()) {`
`353`	`357`	`entry.getValue().shutdown();`
`354`	`358`	`}`
	`359`	`+ this.timeStoreTable.clear();`
`355`	`360`	`log.info("IndexStoreService shutdown gracefully");`
`356`	`361`	`}`
`357`	`362`	`}`
Original file line number	Diff line number	Diff line change
`@@ -74,7 +74,7 @@ public PosixFileSegment(TieredMessageStoreConfig storeConfig,`
`74`	`74`	`String clusterBasePath = TieredStoreUtil.getHash(brokerClusterName) + UNDERLINE + brokerClusterName;`
`75`	`75`	`this.fullPath = Paths.get(basePath, clusterBasePath, filePath,`
`76`	`76`	`fileType.toString(), TieredStoreUtil.offset2FileName(baseOffset)).toString();`
`77`		`- logger.debug("Constructing Posix FileSegment, filePath: {}", fullPath);`
	`77`	`+ logger.info("Constructing Posix FileSegment, filePath: {}", fullPath);`
`78`	`78`
`79`	`79`	`createFile();`
`80`	`80`	`}`