Files
CNC-tools/Indexer.py
T

147 lines
3.9 KiB
Python

#!/usr/bin/env python3
import os
import sys
import pickle
import configparser
from concurrent.futures import ThreadPoolExecutor
# ============================================================
# CONFIG
config = configparser.ConfigParser()
config.read(r"C:\ProgramData\Program-Search\config.ini")
# PATHS
SEARCH_ROOT = config["Paths"]["SEARCH_ROOT"]
INDEX_FILE = config["Paths"]["INDEX_FILE"]
# FEATURES
WORKER_COUNT = config.getint("Features", "WORKER_COUNT", fallback=8)
# ============================================================
def read_file(path, mtime, size):
try:
with open(path, "r", errors="ignore") as f:
return (
path,
{
"filename": os.path.basename(path).lower(),
"content": f.read().lower(),
"mtime": mtime,
"size": size,
}
)
except Exception:
return None
def scan_files(root):
"""Return {path: (mtime, size)} for every file under root."""
found = {}
stack = [root]
while stack:
current = stack.pop()
try:
with os.scandir(current) as it:
for entry in it:
try:
if entry.is_dir(follow_symlinks=False):
stack.append(entry.path)
elif entry.is_file():
st = entry.stat()
found[entry.path] = (st.st_mtime, st.st_size)
except OSError:
pass
except OSError:
pass
return found
def load_index():
try:
with open(INDEX_FILE, "rb") as f:
return pickle.load(f)
except Exception:
return {}
def save_index(index):
os.makedirs(os.path.dirname(INDEX_FILE), exist_ok=True)
# Write to a temp file first so a crash never leaves a corrupt index.
tmp_file = INDEX_FILE + ".tmp"
with open(tmp_file, "wb") as f:
pickle.dump(index, f, protocol=pickle.HIGHEST_PROTOCOL)
os.replace(tmp_file, INDEX_FILE)
def update_index(index, verbose=False):
"""
Bring index up to date with SEARCH_ROOT in place.
Only new or changed files are re-read; deleted files are removed.
Returns True if anything changed.
"""
if verbose:
print("Scanning files...")
on_disk = scan_files(SEARCH_ROOT)
removed = [p for p in index if p not in on_disk]
for p in removed:
del index[p]
to_read = []
for path, (mtime, size) in on_disk.items():
data = index.get(path)
if (
data is None
or data.get("mtime") != mtime
or data.get("size") != size
):
to_read.append((path, mtime, size))
if verbose:
print(
f"Found {len(on_disk)} files: "
f"{len(to_read)} new/changed, {len(removed)} removed"
)
completed = 0
with ThreadPoolExecutor(max_workers=WORKER_COUNT) as executor:
for result in executor.map(lambda a: read_file(*a), to_read):
completed += 1
if verbose and completed % 100 == 0:
print(f"{completed}/{len(to_read)}")
if result:
path, data = result
index[path] = data
return bool(removed or to_read)
def build_index(full=False):
index = {} if full else load_index()
changed = update_index(index, verbose=True)
if not changed and os.path.exists(INDEX_FILE):
print("\nIndex already up to date.")
return
print("\nSaving index...")
save_index(index)
size_mb = os.path.getsize(INDEX_FILE) / 1024 / 1024
print(
f"Index saved:\n"
f"{INDEX_FILE}\n"
f"{len(index)} files\n"
f"{size_mb:.2f} MB"
)
if __name__ == "__main__":
# Run with --full to rebuild the index from scratch.
build_index(full="--full" in sys.argv)