updated programs and added FileManager
This commit is contained in:
1 parent
55177b1444
commit
4c994a28f0
7 files changed
+717
-248
No files matched your search
+98
-54
@@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import sys
|
||||
import pickle
|
||||
import configparser
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
@@ -9,94 +10,137 @@ from concurrent.futures import ThreadPoolExecutor
|
||||
# CONFIG
|
||||
config = configparser.ConfigParser()
|
||||
config.read(r"C:\ProgramData\Program-Search\config.ini")
|
||||
# PATHS
|
||||
SEARCH_ROOT = config["Paths"]["SEARCH_ROOT"]
|
||||
INDEX_FILE = config["Paths"]["INDEX_FILE"]
|
||||
WORKER_COUNT = config.getint("Features", "WORKER_COUNT")
|
||||
# FEATURES
|
||||
WORKER_COUNT = config.getint("Features", "WORKER_COUNT", fallback=8)
|
||||
# ============================================================
|
||||
|
||||
SEARCH_INDEX = {}
|
||||
|
||||
|
||||
def read_file(path):
|
||||
def read_file(path, mtime, size):
|
||||
try:
|
||||
with open(path, "r", errors="ignore") as f:
|
||||
|
||||
return (
|
||||
path,
|
||||
{
|
||||
"filename": os.path.basename(path).lower(),
|
||||
"content": f.read().lower()
|
||||
"content": f.read().lower(),
|
||||
"mtime": mtime,
|
||||
"size": size,
|
||||
}
|
||||
)
|
||||
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def build_index():
|
||||
def scan_files(root):
|
||||
"""Return {path: (mtime, size)} for every file under root."""
|
||||
found = {}
|
||||
stack = [root]
|
||||
|
||||
files = []
|
||||
while stack:
|
||||
current = stack.pop()
|
||||
try:
|
||||
with os.scandir(current) as it:
|
||||
for entry in it:
|
||||
try:
|
||||
if entry.is_dir(follow_symlinks=False):
|
||||
stack.append(entry.path)
|
||||
elif entry.is_file():
|
||||
st = entry.stat()
|
||||
found[entry.path] = (st.st_mtime, st.st_size)
|
||||
except OSError:
|
||||
pass
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
print("Collecting files...")
|
||||
return found
|
||||
|
||||
for root, dirs, filenames in os.walk(SEARCH_ROOT):
|
||||
for filename in filenames:
|
||||
files.append(
|
||||
os.path.join(root, filename)
|
||||
)
|
||||
|
||||
print(f"Found {len(files)} files")
|
||||
print("Building index...\n")
|
||||
def load_index():
|
||||
try:
|
||||
with open(INDEX_FILE, "rb") as f:
|
||||
return pickle.load(f)
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
completed = 0
|
||||
|
||||
with ThreadPoolExecutor(
|
||||
max_workers=WORKER_COUNT
|
||||
) as executor:
|
||||
def save_index(index):
|
||||
os.makedirs(os.path.dirname(INDEX_FILE), exist_ok=True)
|
||||
|
||||
for result in executor.map(
|
||||
read_file,
|
||||
files
|
||||
# Write to a temp file first so a crash never leaves a corrupt index.
|
||||
tmp_file = INDEX_FILE + ".tmp"
|
||||
with open(tmp_file, "wb") as f:
|
||||
pickle.dump(index, f, protocol=pickle.HIGHEST_PROTOCOL)
|
||||
os.replace(tmp_file, INDEX_FILE)
|
||||
|
||||
|
||||
def update_index(index, verbose=False):
|
||||
"""
|
||||
Bring index up to date with SEARCH_ROOT in place.
|
||||
Only new or changed files are re-read; deleted files are removed.
|
||||
Returns True if anything changed.
|
||||
"""
|
||||
if verbose:
|
||||
print("Scanning files...")
|
||||
|
||||
on_disk = scan_files(SEARCH_ROOT)
|
||||
|
||||
removed = [p for p in index if p not in on_disk]
|
||||
for p in removed:
|
||||
del index[p]
|
||||
|
||||
to_read = []
|
||||
for path, (mtime, size) in on_disk.items():
|
||||
data = index.get(path)
|
||||
if (
|
||||
data is None
|
||||
or data.get("mtime") != mtime
|
||||
or data.get("size") != size
|
||||
):
|
||||
to_read.append((path, mtime, size))
|
||||
|
||||
completed += 1
|
||||
|
||||
if completed % 100 == 0:
|
||||
print(
|
||||
f"{completed}/{len(files)}"
|
||||
)
|
||||
|
||||
if result:
|
||||
path, data = result
|
||||
SEARCH_INDEX[path] = data
|
||||
|
||||
print("\nSaving index...")
|
||||
|
||||
os.makedirs(
|
||||
os.path.dirname(INDEX_FILE),
|
||||
exist_ok=True
|
||||
)
|
||||
|
||||
with open(INDEX_FILE, "wb") as f:
|
||||
pickle.dump(
|
||||
SEARCH_INDEX,
|
||||
f,
|
||||
protocol=pickle.HIGHEST_PROTOCOL
|
||||
if verbose:
|
||||
print(
|
||||
f"Found {len(on_disk)} files: "
|
||||
f"{len(to_read)} new/changed, {len(removed)} removed"
|
||||
)
|
||||
|
||||
size_mb = (
|
||||
os.path.getsize(INDEX_FILE)
|
||||
/ 1024
|
||||
/ 1024
|
||||
)
|
||||
completed = 0
|
||||
with ThreadPoolExecutor(max_workers=WORKER_COUNT) as executor:
|
||||
for result in executor.map(lambda a: read_file(*a), to_read):
|
||||
completed += 1
|
||||
if verbose and completed % 100 == 0:
|
||||
print(f"{completed}/{len(to_read)}")
|
||||
if result:
|
||||
path, data = result
|
||||
index[path] = data
|
||||
|
||||
return bool(removed or to_read)
|
||||
|
||||
|
||||
def build_index(full=False):
|
||||
index = {} if full else load_index()
|
||||
|
||||
changed = update_index(index, verbose=True)
|
||||
|
||||
if not changed and os.path.exists(INDEX_FILE):
|
||||
print("\nIndex already up to date.")
|
||||
return
|
||||
|
||||
print("\nSaving index...")
|
||||
save_index(index)
|
||||
|
||||
size_mb = os.path.getsize(INDEX_FILE) / 1024 / 1024
|
||||
print(
|
||||
f"Index saved:\n"
|
||||
f"{INDEX_FILE}\n"
|
||||
f"{len(SEARCH_INDEX)} files\n"
|
||||
f"{len(index)} files\n"
|
||||
f"{size_mb:.2f} MB"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
build_index()
|
||||
# Run with --full to rebuild the index from scratch.
|
||||
build_index(full="--full" in sys.argv)
|
||||
Reference in new issue
Block a user