147 lines
3.9 KiB
Python
147 lines
3.9 KiB
Python
#!/usr/bin/env python3
|
|
|
|
import os
|
|
import sys
|
|
import pickle
|
|
import configparser
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
|
|
# ============================================================
|
|
# CONFIG
|
|
config = configparser.ConfigParser()
|
|
config.read(r"C:\ProgramData\Program-Search\config.ini")
|
|
# PATHS
|
|
SEARCH_ROOT = config["Paths"]["SEARCH_ROOT"]
|
|
INDEX_FILE = config["Paths"]["INDEX_FILE"]
|
|
# FEATURES
|
|
WORKER_COUNT = config.getint("Features", "WORKER_COUNT", fallback=8)
|
|
# ============================================================
|
|
|
|
|
|
def read_file(path, mtime, size):
|
|
try:
|
|
with open(path, "r", errors="ignore") as f:
|
|
return (
|
|
path,
|
|
{
|
|
"filename": os.path.basename(path).lower(),
|
|
"content": f.read().lower(),
|
|
"mtime": mtime,
|
|
"size": size,
|
|
}
|
|
)
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def scan_files(root):
|
|
"""Return {path: (mtime, size)} for every file under root."""
|
|
found = {}
|
|
stack = [root]
|
|
|
|
while stack:
|
|
current = stack.pop()
|
|
try:
|
|
with os.scandir(current) as it:
|
|
for entry in it:
|
|
try:
|
|
if entry.is_dir(follow_symlinks=False):
|
|
stack.append(entry.path)
|
|
elif entry.is_file():
|
|
st = entry.stat()
|
|
found[entry.path] = (st.st_mtime, st.st_size)
|
|
except OSError:
|
|
pass
|
|
except OSError:
|
|
pass
|
|
|
|
return found
|
|
|
|
|
|
def load_index():
|
|
try:
|
|
with open(INDEX_FILE, "rb") as f:
|
|
return pickle.load(f)
|
|
except Exception:
|
|
return {}
|
|
|
|
|
|
def save_index(index):
|
|
os.makedirs(os.path.dirname(INDEX_FILE), exist_ok=True)
|
|
|
|
# Write to a temp file first so a crash never leaves a corrupt index.
|
|
tmp_file = INDEX_FILE + ".tmp"
|
|
with open(tmp_file, "wb") as f:
|
|
pickle.dump(index, f, protocol=pickle.HIGHEST_PROTOCOL)
|
|
os.replace(tmp_file, INDEX_FILE)
|
|
|
|
|
|
def update_index(index, verbose=False):
|
|
"""
|
|
Bring index up to date with SEARCH_ROOT in place.
|
|
Only new or changed files are re-read; deleted files are removed.
|
|
Returns True if anything changed.
|
|
"""
|
|
if verbose:
|
|
print("Scanning files...")
|
|
|
|
on_disk = scan_files(SEARCH_ROOT)
|
|
|
|
removed = [p for p in index if p not in on_disk]
|
|
for p in removed:
|
|
del index[p]
|
|
|
|
to_read = []
|
|
for path, (mtime, size) in on_disk.items():
|
|
data = index.get(path)
|
|
if (
|
|
data is None
|
|
or data.get("mtime") != mtime
|
|
or data.get("size") != size
|
|
):
|
|
to_read.append((path, mtime, size))
|
|
|
|
if verbose:
|
|
print(
|
|
f"Found {len(on_disk)} files: "
|
|
f"{len(to_read)} new/changed, {len(removed)} removed"
|
|
)
|
|
|
|
completed = 0
|
|
with ThreadPoolExecutor(max_workers=WORKER_COUNT) as executor:
|
|
for result in executor.map(lambda a: read_file(*a), to_read):
|
|
completed += 1
|
|
if verbose and completed % 100 == 0:
|
|
print(f"{completed}/{len(to_read)}")
|
|
if result:
|
|
path, data = result
|
|
index[path] = data
|
|
|
|
return bool(removed or to_read)
|
|
|
|
|
|
def build_index(full=False):
|
|
index = {} if full else load_index()
|
|
|
|
changed = update_index(index, verbose=True)
|
|
|
|
if not changed and os.path.exists(INDEX_FILE):
|
|
print("\nIndex already up to date.")
|
|
return
|
|
|
|
print("\nSaving index...")
|
|
save_index(index)
|
|
|
|
size_mb = os.path.getsize(INDEX_FILE) / 1024 / 1024
|
|
print(
|
|
f"Index saved:\n"
|
|
f"{INDEX_FILE}\n"
|
|
f"{len(index)} files\n"
|
|
f"{size_mb:.2f} MB"
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
# Run with --full to rebuild the index from scratch.
|
|
build_index(full="--full" in sys.argv)
|