feat: added bigger buffer size

kris-dev-hub · Dec 25, 2023 · 0a00013 · 0a00013
1 parent 4588ca1
commit 0a00013
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/pkg/commoncrawl/wat.go b/pkg/commoncrawl/wat.go
@@ -268,7 +268,7 @@ func ParseWatByLine(filePath string, linkFile string, pageFile string, savePage
 	pageMap := make(map[string]FilePage)
 	linkMap := make(map[string]FileLink)
 
-	const maxCapacityScanner = 3 * 1024 * 1024 // 3*1MB
+	const maxCapacityScanner = 5 * 1024 * 1024 // 5*1MB
 
 	// Open the .gz file
 	file, err := os.Open(filePath)