Files
CNC-tools/Indexer.py
T

102 lines
2.1 KiB
Python

#!/usr/bin/env python3
import os
import pickle
import configparser
from concurrent.futures import ThreadPoolExecutor
# ============================================================
# CONFIG
config = configparser.ConfigParser()
config.read(r"C:\ProgramData\Program-Search\config.ini")
SEARCH_ROOT = config["Paths"]["SEARCH_ROOT"]
INDEX_FILE = config["Paths"]["INDEX_FILE"]
WORKER_COUNT = config.getint("Features", "WORKER_COUNT")
# ============================================================
SEARCH_INDEX = {}
def read_file(path):
try:
with open(path, "r", errors="ignore") as f:
return (
path,
{
"filename": os.path.basename(path).lower(),
"content": f.read().lower()
}
)
except Exception:
return None
def build_index():
files = []
print("Collecting files...")
for root, dirs, filenames in os.walk(SEARCH_ROOT):
for filename in filenames:
files.append(
os.path.join(root, filename)
)
print(f"Found {len(files)} files")
print("Building index...\n")
completed = 0
with ThreadPoolExecutor(
max_workers=WORKER_COUNT
) as executor:
for result in executor.map(
read_file,
files
):
completed += 1
if completed % 100 == 0:
print(
f"{completed}/{len(files)}"
)
if result:
path, data = result
SEARCH_INDEX[path] = data
print("\nSaving index...")
os.makedirs(
os.path.dirname(INDEX_FILE),
exist_ok=True
)
with open(INDEX_FILE, "wb") as f:
pickle.dump(
SEARCH_INDEX,
f,
protocol=pickle.HIGHEST_PROTOCOL
)
size_mb = (
os.path.getsize(INDEX_FILE)
/ 1024
/ 1024
)
print(
f"Index saved:\n"
f"{INDEX_FILE}\n"
f"{len(SEARCH_INDEX)} files\n"
f"{size_mb:.2f} MB"
)
if __name__ == "__main__":
build_index()