Updated Search.py to use an index.
This commit is contained in:
1 parent
79d9fbf09a
commit
199d622c24
3 files changed
+335
-71
No files matched your search
+102
@@ -0,0 +1,102 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import pickle
|
||||
import configparser
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
# ============================================================
|
||||
# CONFIG
|
||||
config = configparser.ConfigParser()
|
||||
config.read(r"C:\ProgramData\Program-Search\config.ini")
|
||||
SEARCH_ROOT = config["Paths"]["SEARCH_ROOT"]
|
||||
INDEX_FILE = config["Paths"]["INDEX_FILE"]
|
||||
WORKER_COUNT = config.getint("Features", "WORKER_COUNT")
|
||||
# ============================================================
|
||||
|
||||
SEARCH_INDEX = {}
|
||||
|
||||
|
||||
def read_file(path):
|
||||
try:
|
||||
with open(path, "r", errors="ignore") as f:
|
||||
|
||||
return (
|
||||
path,
|
||||
{
|
||||
"filename": os.path.basename(path).lower(),
|
||||
"content": f.read().lower()
|
||||
}
|
||||
)
|
||||
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def build_index():
|
||||
|
||||
files = []
|
||||
|
||||
print("Collecting files...")
|
||||
|
||||
for root, dirs, filenames in os.walk(SEARCH_ROOT):
|
||||
for filename in filenames:
|
||||
files.append(
|
||||
os.path.join(root, filename)
|
||||
)
|
||||
|
||||
print(f"Found {len(files)} files")
|
||||
print("Building index...\n")
|
||||
|
||||
completed = 0
|
||||
|
||||
with ThreadPoolExecutor(
|
||||
max_workers=WORKER_COUNT
|
||||
) as executor:
|
||||
|
||||
for result in executor.map(
|
||||
read_file,
|
||||
files
|
||||
):
|
||||
|
||||
completed += 1
|
||||
|
||||
if completed % 100 == 0:
|
||||
print(
|
||||
f"{completed}/{len(files)}"
|
||||
)
|
||||
|
||||
if result:
|
||||
path, data = result
|
||||
SEARCH_INDEX[path] = data
|
||||
|
||||
print("\nSaving index...")
|
||||
|
||||
os.makedirs(
|
||||
os.path.dirname(INDEX_FILE),
|
||||
exist_ok=True
|
||||
)
|
||||
|
||||
with open(INDEX_FILE, "wb") as f:
|
||||
pickle.dump(
|
||||
SEARCH_INDEX,
|
||||
f,
|
||||
protocol=pickle.HIGHEST_PROTOCOL
|
||||
)
|
||||
|
||||
size_mb = (
|
||||
os.path.getsize(INDEX_FILE)
|
||||
/ 1024
|
||||
/ 1024
|
||||
)
|
||||
|
||||
print(
|
||||
f"Index saved:\n"
|
||||
f"{INDEX_FILE}\n"
|
||||
f"{len(SEARCH_INDEX)} files\n"
|
||||
f"{size_mb:.2f} MB"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
build_index()
|
||||
Reference in new issue
Block a user