Feature: add pattern scraper and optimize batch export

- Added Pull Pattern Images tool with parallel background downloading

- Optimized Export Folder Stats to run headlessly (massive speedup)

- Dynamically name exported CSVs based on source folder

- Fixed German CSV localization and UTF-8 BOM

- Updated README and walkthrough
This commit is contained in:
lukas 2026-03-10 17:28:15 +01:00
parent 635b65b7e1
commit ac79d0e5dc
5 changed files with 262 additions and 19 deletions

View File

@ -69,3 +69,11 @@
"menu.edit" = "Bearbeiten"
"menu.view" = "Ansicht"
"menu.tools" = "Werkzeuge"
"toolbar.pull_patterns" = "Muster-Bilder herunterladen"
"dialog.puller_title" = "Muster-Bilder herunterladen"
"dialog.puller_instruction" = "CSGOSkins.gg Artikel-URL einfügen:"
"dialog.puller_start" = "Download starten"
"dialog.puller_cancel" = "Abbrechen"
"dialog.puller_invalid_url" = "Ungültiges URL-Format."
"dialog.puller_success" = "Alle Muster erfolgreich heruntergeladen!"

View File

@ -69,3 +69,11 @@
"menu.edit" = "Edit"
"menu.view" = "View"
"menu.tools" = "Tools"
"toolbar.pull_patterns" = "Pull Pattern Images"
"dialog.puller_title" = "Pull Pattern Images"
"dialog.puller_instruction" = "Paste a CSGOSkins.gg item URL:"
"dialog.puller_start" = "Start Download"
"dialog.puller_cancel" = "Cancel"
"dialog.puller_invalid_url" = "Invalid URL format."
"dialog.puller_success" = "All patterns downloaded successfully!"

View File

@ -233,6 +233,50 @@ class QtImageProcessor:
total_excl=total_excl,
)
def get_stats_headless(self, image: Image.Image) -> Stats:
"""Calculate color-match statistics natively without building UI elements or scaling."""
base = image.convert("RGBA")
arr = np.asarray(base, dtype=np.float32)
rgb = arr[..., :3] / 255.0
alpha_ch = arr[..., 3]
hsv = _rgb_to_hsv_numpy(rgb)
hue = hsv[..., 0]
sat = hsv[..., 1]
val = hsv[..., 2]
hue_min = float(self.hue_min)
hue_max = float(self.hue_max)
if hue_min <= hue_max:
hue_ok = (hue >= hue_min) & (hue <= hue_max)
else:
hue_ok = (hue >= hue_min) | (hue <= hue_max)
match_mask = (
hue_ok
& (sat >= float(self.sat_min))
& (val >= float(self.val_min))
& (val <= float(self.val_max))
& (alpha_ch > 0)
)
excl_mask = self._build_exclusion_mask_numpy(base.size)
keep_match = match_mask & ~excl_mask
excl_match = match_mask & excl_mask
visible = alpha_ch > 0
return Stats(
matches_all=int(match_mask[visible].sum()),
total_all=int(visible.sum()),
matches_keep=int(keep_match[visible].sum()),
total_keep=int((visible & ~excl_mask).sum()),
matches_excl=int(excl_match[visible].sum()),
total_excl=int((visible & excl_mask).sum()),
)
# helpers ----------------------------------------------------------------
def _matches(self, r: int, g: int, b: int) -> bool:

View File

@ -12,6 +12,7 @@ from PySide6 import QtCore, QtGui, QtWidgets
from app.i18n import I18nMixin
from app.logic import SUPPORTED_IMAGE_EXTENSIONS
from .image_processor import QtImageProcessor
from .pattern_puller import PatternPullerDialog
DEFAULT_COLOR = "#763e92"
@ -627,6 +628,8 @@ class MainWindow(QtWidgets.QMainWindow, I18nMixin):
tools_menu.addAction("🎨 " + self._t("toolbar.choose_color"), lambda: self._invoke_action("choose_color"))
tools_menu.addAction("🖱 " + self._t("toolbar.pick_from_image"), lambda: self._invoke_action("pick_from_image"))
tools_menu.addAction("" + self._t("toolbar.toggle_free_draw"), lambda: self._invoke_action("toggle_free_draw"))
tools_menu.addSeparator()
tools_menu.addAction("📥 " + self._t("toolbar.pull_patterns"), lambda: self._invoke_action("pull_patterns"))
# View Menu
view_menu = self.menu_bar.addMenu(self._t("menu.view"))
@ -750,6 +753,7 @@ class MainWindow(QtWidgets.QMainWindow, I18nMixin):
"toggle_theme": self.toggle_theme,
"show_previous_image": self.show_previous_image,
"show_next_image": self.show_next_image,
"pull_patterns": self.open_pattern_puller,
}
def _invoke_action(self, key: str) -> None:
@ -801,10 +805,13 @@ class MainWindow(QtWidgets.QMainWindow, I18nMixin):
QtWidgets.QMessageBox.information(self, self._t("dialog.info_title"), self._t("dialog.no_image_loaded"))
return
folder_path = self.processor.preview_paths[0].parent
default_filename = f"icra_stats_{folder_path.name}.csv"
csv_path, _ = QtWidgets.QFileDialog.getSaveFileName(
self,
self._t("dialog.export_stats_title"),
str(self.processor.preview_paths[0].parent / "icra_stats.csv"),
str(folder_path / default_filename),
self._t("dialog.csv_filter")
)
if not csv_path:
@ -812,7 +819,11 @@ class MainWindow(QtWidgets.QMainWindow, I18nMixin):
total = len(self.processor.preview_paths)
is_eu = self.language == "de"
# Determine localization: German language or DE system locale defaults to EU format (; delimiter, , decimal)
import locale
sys_lang = QtCore.QLocale().name().split('_')[0].lower()
is_eu = (self.language == "de" or sys_lang == "de")
delimiter = ";" if is_eu else ","
decimal = "," if is_eu else "."
@ -829,15 +840,9 @@ class MainWindow(QtWidgets.QMainWindow, I18nMixin):
self.status_label.setText(self._t("status.exporting", current=str(i+1), total=str(total)))
QtWidgets.QApplication.processEvents() # Keep UI vaguely responsive
# Process without modifying the UI current_index
# Process purely mathematically without breaking the active UI context
img = Image.open(img_path)
old_orig = self.processor.orig_img
old_preview = self.processor.preview_img
self.processor.orig_img = img
self.processor._build_preview()
self.processor._rebuild_overlay()
s = self.processor.stats
s = self.processor.get_stats_headless(img)
pct_all = (s.matches_all / s.total_all * 100) if s.total_all else 0.0
pct_keep = (s.matches_keep / s.total_keep * 100) if s.total_keep else 0.0
@ -856,18 +861,16 @@ class MainWindow(QtWidgets.QMainWindow, I18nMixin):
])
img.close()
# Restore previous state
self.processor.orig_img = old_orig
self.processor.preview_img = old_preview
# Compute max width per column for alignment, plus extra space so it's not cramped
col_widths = [max(len(str(item)) for item in col) + 4 for col in zip(*rows)]
with open(csv_path, mode="w", newline="", encoding="utf-8") as f:
writer = csv.writer(f, delimiter=delimiter)
# Excel on Windows prefers utf-8-sig (with BOM) to identify the encoding correctly.
with open(csv_path, mode="w", newline="", encoding="utf-8-sig") as f:
for row in rows:
padded_row = [f"{str(item):>{width}}" for item, width in zip(row, col_widths)]
writer.writerow(padded_row)
# Manual formatting to support both alignment for text editors AND valid CSV for Excel.
# We pad the strings but keep the delimiter clean.
padded_cells = [f"{str(item):>{width}}" for item, width in zip(row, col_widths)]
f.write(delimiter.join(padded_cells) + "\n")
# Restore overlay state for currently viewed image
self.processor._rebuild_overlay()
@ -1002,9 +1005,12 @@ class MainWindow(QtWidgets.QMainWindow, I18nMixin):
self.status_label.setText(
self._t("status.pick_mode_from_image", hue=hue, saturation=sat, value=val)
)
self._exit_pick_mode()
self._refresh_overlay_only()
def open_pattern_puller(self) -> None:
dialog = PatternPullerDialog(self.language, parent=self)
dialog.exec()
# Drag-and-drop ----------------------------------------------------------
def dragEnterEvent(self, event: QtGui.QDragEnterEvent) -> None:

177
app/qt/pattern_puller.py Normal file
View File

@ -0,0 +1,177 @@
"""Dialog and worker thread for batch downloading CSGOSkins patterns."""
from __future__ import annotations
import re
import time
import urllib.request
import urllib.error
from pathlib import Path
from PySide6 import QtCore, QtWidgets, QtGui
from app.i18n import I18nMixin
import concurrent.futures
class PatternDownloadWorker(QtCore.QThread):
progress = QtCore.Signal(int, int) # current, total
status = QtCore.Signal(str) # textual update
finished = QtCore.Signal(bool) # True if Success, False if Interrupted/Error
error = QtCore.Signal(str) # Error message
def __init__(self, slug: str, save_dir: Path, parent: QtCore.QObject | None = None) -> None:
super().__init__(parent)
self.slug = slug
self.save_dir = save_dir
self.total_seeds = 1000
def _download_seed(self, seed: int) -> tuple[bool, str | None]:
url = f"https://cdn.csgoskins.gg/public/images/patterns/v1/{self.slug}/{seed}.png"
filename = self.save_dir / f"{seed}.png"
if filename.exists():
return True, None
try:
req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0 ICRA/1.0'})
with urllib.request.urlopen(req, timeout=10) as response:
with open(filename, 'wb') as f:
f.write(response.read())
return True, None
except urllib.error.HTTPError as e:
return False, f"HTTP {e.code}"
except Exception as e:
return False, f"Network error: {e}"
def run(self) -> None:
self.save_dir.mkdir(parents=True, exist_ok=True)
completed = 0
# Validate seed 1 synchronously first to avoid spawning 1000 threads for invalid slugs
success, error_msg = self._download_seed(1)
if not success and error_msg in ("HTTP 403", "HTTP 404"):
self.error.emit(f"Failed to fetch seed 1. Does '{self.slug}' have patterns?")
self.finished.emit(False)
return
completed += 1
self.progress.emit(completed, self.total_seeds)
# Download the rest concurrently
with concurrent.futures.ThreadPoolExecutor(max_workers=20) as executor:
future_to_seed = {executor.submit(self._download_seed, seed): seed for seed in range(2, self.total_seeds + 1)}
for future in concurrent.futures.as_completed(future_to_seed):
if self.isInterruptionRequested():
self.status.emit("Download cancelled. Waiting for threads to finish...")
executor.shutdown(wait=False, cancel_futures=True)
self.finished.emit(False)
return
seed = future_to_seed[future]
completed += 1
success, error = future.result()
if success:
self.status.emit(f"Downloaded seed {seed}/{self.total_seeds}")
else:
self.status.emit(f"Skipped seed {seed} ({error})")
self.progress.emit(completed, self.total_seeds)
self.status.emit("Download complete!")
self.finished.emit(True)
class PatternPullerDialog(QtWidgets.QDialog, I18nMixin):
"""Dialog for extracting patterns from CSGOSkins.gg URLs."""
def __init__(self, language: str, parent: QtWidgets.QWidget | None = None) -> None:
super().__init__(parent)
self.init_i18n(language)
self.setWindowTitle(self._t("dialog.puller_title", default="Pull Pattern Images"))
self.setMinimumWidth(450)
self._worker: PatternDownloadWorker | None = None
self._build_ui()
def _build_ui(self) -> None:
layout = QtWidgets.QVBoxLayout(self)
layout.setSpacing(12)
instruction_label = QtWidgets.QLabel(self._t("dialog.puller_instruction", default="Paste a CSGOSkins.gg item URL:"))
layout.addWidget(instruction_label)
self.url_input = QtWidgets.QLineEdit()
self.url_input.setPlaceholderText("https://csgoskins.gg/items/glock-18-trace-lock")
layout.addWidget(self.url_input)
self.status_label = QtWidgets.QLabel("")
self.status_label.setStyleSheet("color: palette(window-text); font-style: italic;")
layout.addWidget(self.status_label)
self.progress_bar = QtWidgets.QProgressBar()
self.progress_bar.setRange(0, 1000)
self.progress_bar.setValue(0)
layout.addWidget(self.progress_bar)
button_layout = QtWidgets.QHBoxLayout()
self.start_btn = QtWidgets.QPushButton(self._t("dialog.puller_start", default="Start Download"))
self.cancel_btn = QtWidgets.QPushButton(self._t("dialog.puller_cancel", default="Cancel"))
self.start_btn.clicked.connect(self._on_start_clicked)
self.cancel_btn.clicked.connect(self._on_cancel_clicked)
button_layout.addWidget(self.start_btn)
button_layout.addWidget(self.cancel_btn)
layout.addLayout(button_layout)
def _extract_slug(self, url: str) -> str | None:
# Match https://csgoskins.gg/items/SLUG
match = re.search(r"csgoskins\.gg/items/([^/?#]+)", url)
if match:
return match.group(1).lower()
return None
def _on_start_clicked(self) -> None:
url = self.url_input.text().strip()
slug = self._extract_slug(url)
if not slug:
QtWidgets.QMessageBox.warning(self, "Error", self._t("dialog.puller_invalid_url", default="Invalid URL format."))
return
save_dir = Path("images") / slug
self.start_btn.setEnabled(False)
self.url_input.setEnabled(False)
self.progress_bar.setValue(0)
self.status_label.setText("Starting download...")
self._worker = PatternDownloadWorker(slug=slug, save_dir=save_dir, parent=self)
self._worker.progress.connect(self._on_progress)
self._worker.status.connect(self.status_label.setText)
self._worker.error.connect(self._on_error)
self._worker.finished.connect(self._on_finished)
self._worker.start()
def _on_cancel_clicked(self) -> None:
if self._worker and self._worker.isRunning():
self._worker.requestInterruption()
self.cancel_btn.setEnabled(False)
self.status_label.setText("Cancelling...")
else:
self.reject() # Close dialog if not downloading
def _on_progress(self, current: int, total: int) -> None:
self.progress_bar.setValue(current)
def _on_error(self, message: str) -> None:
QtWidgets.QMessageBox.warning(self, "Error", message)
def _on_finished(self, success: bool) -> None:
self.start_btn.setEnabled(True)
self.url_input.setEnabled(True)
self.cancel_btn.setEnabled(True)
self._worker = None
if success:
QtWidgets.QMessageBox.information(self, "Done", self._t("dialog.puller_success", default="All patterns downloaded successfully!"))