102 lines
2.1 KiB
Python
102 lines
2.1 KiB
Python
#!/usr/bin/env python3
|
|
|
|
import os
|
|
import pickle
|
|
import configparser
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
|
|
# ============================================================
|
|
# CONFIG
|
|
config = configparser.ConfigParser()
|
|
config.read(r"C:\ProgramData\Program-Search\config.ini")
|
|
SEARCH_ROOT = config["Paths"]["SEARCH_ROOT"]
|
|
INDEX_FILE = config["Paths"]["INDEX_FILE"]
|
|
WORKER_COUNT = config.getint("Features", "WORKER_COUNT")
|
|
# ============================================================
|
|
|
|
SEARCH_INDEX = {}
|
|
|
|
|
|
def read_file(path):
|
|
try:
|
|
with open(path, "r", errors="ignore") as f:
|
|
|
|
return (
|
|
path,
|
|
{
|
|
"filename": os.path.basename(path).lower(),
|
|
"content": f.read().lower()
|
|
}
|
|
)
|
|
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def build_index():
|
|
|
|
files = []
|
|
|
|
print("Collecting files...")
|
|
|
|
for root, dirs, filenames in os.walk(SEARCH_ROOT):
|
|
for filename in filenames:
|
|
files.append(
|
|
os.path.join(root, filename)
|
|
)
|
|
|
|
print(f"Found {len(files)} files")
|
|
print("Building index...\n")
|
|
|
|
completed = 0
|
|
|
|
with ThreadPoolExecutor(
|
|
max_workers=WORKER_COUNT
|
|
) as executor:
|
|
|
|
for result in executor.map(
|
|
read_file,
|
|
files
|
|
):
|
|
|
|
completed += 1
|
|
|
|
if completed % 100 == 0:
|
|
print(
|
|
f"{completed}/{len(files)}"
|
|
)
|
|
|
|
if result:
|
|
path, data = result
|
|
SEARCH_INDEX[path] = data
|
|
|
|
print("\nSaving index...")
|
|
|
|
os.makedirs(
|
|
os.path.dirname(INDEX_FILE),
|
|
exist_ok=True
|
|
)
|
|
|
|
with open(INDEX_FILE, "wb") as f:
|
|
pickle.dump(
|
|
SEARCH_INDEX,
|
|
f,
|
|
protocol=pickle.HIGHEST_PROTOCOL
|
|
)
|
|
|
|
size_mb = (
|
|
os.path.getsize(INDEX_FILE)
|
|
/ 1024
|
|
/ 1024
|
|
)
|
|
|
|
print(
|
|
f"Index saved:\n"
|
|
f"{INDEX_FILE}\n"
|
|
f"{len(SEARCH_INDEX)} files\n"
|
|
f"{size_mb:.2f} MB"
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
build_index() |