238cd9c6c3
- Authoritative 30 Hz server (ENet, SQLite): accounts, characters, wild AI, tall grass / water / wandering spawns, fishing, captures, follower chains, clock + weather, LAN beacon - Client: title / join / character creator / play screens, prediction + reconciliation, interpolated players and parties, HD-2D UI (dialogue with Professor Henry, HUD, collection) - Shared deterministic zone generator, movement, protocol, roster, appearance - Paper-doll character art pipeline (PixelLab base walk + per-item animation edits) - Brandling roster from the company list (logo colors, jev types), Blender safari props - Headless test bot
219 lines
10 KiB
Python
219 lines
10 KiB
Python
"""Read the web-design client export (ClickUp CSV) into a clean company list.
|
||
|
||
Only the company name, its domain, the staging-site URL (used to look for a logo, never
|
||
stored), the services / key-highlights text (for type classification) and the colour words
|
||
of the logo note are extracted. Every other field (logins, contacts, addresses, client names)
|
||
is ignored.
|
||
"""
|
||
import csv
|
||
import re
|
||
import unicodedata
|
||
from dataclasses import dataclass, field
|
||
|
||
csv.field_size_limit(1 << 30)
|
||
|
||
# Lines that start a new section of the task text (end of a services / highlights list).
|
||
SECTION = re.compile(r"^(additional information|key highlights|account manager|client name|business address|"
|
||
r"domain|site pages|logo|contact|login|theme|target city|business hours|phone|email|"
|
||
r"do we have|uploaded logo|client desired|if “other”|if \"other\")\b", re.I)
|
||
BOILERPLATE = {"locally owned", "family owned", "reliable", "free estimates", "free estimate", "licensed",
|
||
"insured", "licensed and insured", "licensed & insured", "veteran owned", "woman owned",
|
||
"women owned", "locally owned, family owned", "bonded", "affordable", "trusted",
|
||
"satisfaction guaranteed", "24/7", "emergency services", "competitive pricing", "quality work"}
|
||
PRIVATE = re.compile(r"@|https?:|www\.|\d[\d\s().-]{6,}\d|password|username|login", re.I)
|
||
|
||
LEGAL = re.compile(r"[\s,]+(?:l\.?\s?l\.?\s?c\.?|inc\.?|incorporated|corp\.?|corporation|ltd\.?|limited|"
|
||
r"p\.?\s?l\.?\s?l\.?\s?c\.?|l\.?\s?l\.?\s?p\.?|p\.?\s?c\.?|p\.?\s?a\.?|l\.?c\.?|co\.?)\s*$", re.I)
|
||
NOT_COMPANY = re.compile(r"^(test|testing|template|copy of|untitled|new task|sample|demo|example|tbd|n/?a|"
|
||
r"none|unknown|duplicate|delete|do not use)\b", re.I)
|
||
# Short all-caps words that are real words (title-cased when fixing a shouted name);
|
||
# every other word of 1-3 letters is kept as an acronym (HVAC-style initials).
|
||
SHORT_WORDS = {"the", "and", "of", "for", "all", "pro", "top", "one", "big", "new", "sun", "bay", "oak", "art",
|
||
"car", "dog", "fix", "man", "men", "way", "day", "air", "hot", "red", "sky", "sea", "ace", "job",
|
||
"bee", "bug", "cut", "hub", "go", "my", "on", "to", "in", "at", "by", "a", "an", "or", "us", "we",
|
||
"box", "fox", "elk", "owl", "ant", "pet", "pup", "joy", "sol", "max", "tri", "bro", "now", "get",
|
||
"our", "you", "pine", "key", "inn", "spa", "tub", "pool", "dry", "wet", "mud", "ice", "tile",
|
||
"it", "is", "git", "don", "co", "do", "no", "up", "me", "dad", "mom", "son", "sons",
|
||
"two", "six", "ten", "own", "pal", "rad", "fun", "eco", "zen", "gem", "pop", "wow", "yes", "hey"}
|
||
|
||
COLOR_WORDS = [ # longest phrases first
|
||
("light purple", "#B39DDB"), ("dark purple", "#4A148C"), ("light blue", "#4FC3F7"), ("baby blue", "#81D4FA"),
|
||
("sky blue", "#4FC3F7"), ("royal blue", "#2747C8"), ("dark blue", "#0D2B6B"), ("navy blue", "#1A237E"),
|
||
("light green", "#7CB342"), ("lime green", "#7CB342"), ("dark green", "#1B5E20"), ("forest green", "#1B5E20"),
|
||
("hunter green", "#2E5A3A"), ("dark red", "#8B1A1A"), ("light grey", "#BDBDBD"), ("light gray", "#BDBDBD"),
|
||
("dark grey", "#4A4A4A"), ("dark gray", "#4A4A4A"), ("light pink", "#F8BBD0"), ("hot pink", "#E91E63"),
|
||
("navy", "#1A237E"), ("maroon", "#7B1E2B"), ("burgundy", "#7B1E2B"), ("crimson", "#B71C1C"),
|
||
("red", "#C62828"), ("orange", "#EF6C00"), ("yellow", "#FBC02D"), ("gold", "#D4A017"), ("golden", "#D4A017"),
|
||
("green", "#2E7D32"), ("lime", "#9CCC65"), ("olive", "#6B7A2A"), ("teal", "#00897B"),
|
||
("turquoise", "#26C6DA"), ("aqua", "#26C6DA"), ("cyan", "#00BCD4"), ("blue", "#1565C0"),
|
||
("purple", "#6A1B9A"), ("violet", "#7E57C2"), ("lavender", "#B39DDB"), ("magenta", "#C2185B"),
|
||
("pink", "#EC407A"), ("brown", "#6D4C41"), ("tan", "#C8A97E"), ("beige", "#D7C4A3"), ("cream", "#F3E5AB"),
|
||
("copper", "#B87333"), ("bronze", "#CD7F32"), ("rust", "#B7410E"), ("charcoal", "#36454F"),
|
||
("black", "#212121"), ("white", "#FAFAFA"), ("grey", "#9E9E9E"), ("gray", "#9E9E9E"), ("silver", "#BDBDBD"),
|
||
]
|
||
|
||
|
||
@dataclass
|
||
class Company:
|
||
name: str # display name
|
||
slug: str
|
||
key: str # dedupe key (normalised name)
|
||
domain: str | None = None
|
||
staging: str | None = None # staging site root URL (logo search only; never written out)
|
||
services: list = field(default_factory=list)
|
||
highlights: list = field(default_factory=list)
|
||
colors: list = field(default_factory=list) # hex colours named in the logo note
|
||
has_company_line: bool = False
|
||
richness: tuple = ()
|
||
|
||
|
||
def slugify(s):
|
||
s = unicodedata.normalize("NFKD", s).encode("ascii", "ignore").decode()
|
||
return re.sub(r"[^a-z0-9]+", "-", s.lower()).strip("-")
|
||
|
||
|
||
def _fix_case(name):
|
||
"""Title-case a shouted (ALL-CAPS) or all-lowercase name. In a shouted name, runs of 1-3
|
||
letters stay uppercase (initials like KCS, HVAC-style) unless they are common words."""
|
||
letters = [c for c in name if c.isalpha()]
|
||
if not letters:
|
||
return name
|
||
shouted = all(c.isupper() for c in letters) and len(letters) >= 4 and \
|
||
any(len(w) >= 4 for w in re.findall(r"[^\W\d_]+", name))
|
||
if not (shouted or all(c.islower() for c in letters)):
|
||
return name
|
||
|
||
def run(m):
|
||
w = m.group(0)
|
||
if shouted and len(w) <= 3 and w.lower() not in SHORT_WORDS:
|
||
return w
|
||
return w[:1].upper() + w[1:].lower()
|
||
|
||
name = re.sub(r"[^\W\d_]+", run, name)
|
||
return re.sub(r"(['’])S\b", r"\1s", name)
|
||
|
||
|
||
def clean_name(raw):
|
||
"""Display name: drop legal suffixes and quotes, fix ALL-CAPS or all-lowercase names."""
|
||
s = unicodedata.normalize("NFKC", raw or "").replace("\u00a0", " ")
|
||
s = re.sub(r"\s+", " ", s).strip().strip("\"'“”*").strip()
|
||
s = re.sub(r"\s*\((?:redesign|old|new|copy|duplicate|dupe|2nd|second|rebuild|canceled|cancelled)[^)]*\)\s*$", "", s, flags=re.I)
|
||
for _ in range(3):
|
||
m = LEGAL.search(s)
|
||
if not m:
|
||
break
|
||
rest = s[:m.start()].rstrip(" ,&-")
|
||
# "Co." only goes when a real name is left ("Health Co." keeps it)
|
||
if re.fullmatch(r"[\s,]+co\.?\s*", m.group(0), re.I) and len(rest.split()) < 2:
|
||
break
|
||
s = rest
|
||
s = s.strip(" ,.-&")
|
||
return _fix_case(s)
|
||
|
||
|
||
def _value(text, label):
|
||
m = re.search(rf"^\s*{label}\s*:\s*(.*)$", text, re.I | re.M)
|
||
return m.group(1).strip() if m else ""
|
||
|
||
|
||
def _section(lines, start_pat, limit=14):
|
||
out, on = [], False
|
||
for line in lines:
|
||
s = line.strip()
|
||
if not on:
|
||
if re.match(start_pat, s, re.I):
|
||
on = True
|
||
rest = re.sub(start_pat, "", s, flags=re.I).strip(" :")
|
||
if rest:
|
||
out.append(rest)
|
||
continue
|
||
if not s or SECTION.match(s):
|
||
if out or SECTION.match(s):
|
||
break
|
||
continue
|
||
out.append(s)
|
||
if len(out) >= limit:
|
||
break
|
||
items = []
|
||
for s in out:
|
||
s = re.sub(r"^[-•*·\d.)\s]+", "", s).strip(" ,;")
|
||
if not s or PRIVATE.search(s) or len(s) > 80:
|
||
continue
|
||
items.append(s)
|
||
return items
|
||
|
||
|
||
def _domain(text):
|
||
"""The client's desired domain, else the site being redesigned. clickwise.dev never counts."""
|
||
from logo_fetch import normalize_domain # Company_Sprite_Gen (on sys.path)
|
||
values = [_value(text, "Client desired domain name")]
|
||
m = re.search(r"is a redesign of\s*:?\s*(\S+)", text, re.I)
|
||
if m:
|
||
values.append(m.group(1))
|
||
for v in values:
|
||
for tok in re.findall(r"(?:https?://)?(?:www\.)?[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)*\.[A-Za-z]{2,}", v):
|
||
d = normalize_domain(tok)
|
||
if d and not d.endswith("clickwise.dev") and not d.endswith("jotform.com"):
|
||
return d
|
||
return None
|
||
|
||
|
||
def _colors(text):
|
||
note = _value(text, r"Do we have their logo\? If not what needs to be done")
|
||
note = note.lower()
|
||
found = []
|
||
for word, hexv in COLOR_WORDS:
|
||
for m in re.finditer(rf"\b{re.escape(word)}\b", note):
|
||
found.append((m.start(), word, hexv))
|
||
note = note[:m.start()] + " " * len(word) + note[m.end():]
|
||
return [h for _, _, h in sorted(found)][:3]
|
||
|
||
|
||
def parse_row(row):
|
||
text = (row.get("Task Content") or "").replace("\\n", "\n")
|
||
company_line = _value(text, "Company name")
|
||
raw = company_line if company_line and company_line.upper() not in ("NA", "N/A") else row.get("Task Name", "")
|
||
name = clean_name(raw)
|
||
if not name or sum(ch.isalpha() for ch in name) < 3 or NOT_COMPANY.match(name) or not slugify(name):
|
||
return None
|
||
lines = text.split("\n")
|
||
staging = None
|
||
m = re.search(r"https?://([a-z0-9-]+)\.clickwise\.dev", text, re.I)
|
||
if m:
|
||
staging = f"https://{m.group(1).lower()}.clickwise.dev/"
|
||
services = _section(lines, r"^\(?services\)?\s*:?")
|
||
highlights = [h for h in _section(lines, r"^key highlights\s*:?", limit=8) if h.lower() not in BOILERPLATE]
|
||
c = Company(name=name, slug=slugify(name), key=re.sub(r"^the-", "", slugify(name)).replace("-", ""),
|
||
domain=_domain(text), staging=staging, services=services, highlights=highlights,
|
||
colors=_colors(text), has_company_line=bool(company_line))
|
||
c.richness = (c.has_company_line, bool(c.domain), len(c.services), bool(c.staging), len(text))
|
||
return c
|
||
|
||
|
||
def load_companies(csv_path):
|
||
"""(companies sorted by slug, stats). Duplicates (same normalised name) keep the richest
|
||
row, filling its missing domain / staging / colours from the others."""
|
||
stats = {"rows": 0, "not_company": 0, "duplicates": 0}
|
||
by_key = {}
|
||
with open(csv_path, newline="", encoding="utf-8") as f:
|
||
for row in csv.DictReader(f):
|
||
stats["rows"] += 1
|
||
c = parse_row(row)
|
||
if c is None:
|
||
stats["not_company"] += 1
|
||
continue
|
||
old = by_key.get(c.key)
|
||
if old is None:
|
||
by_key[c.key] = c
|
||
continue
|
||
stats["duplicates"] += 1
|
||
best, other = (c, old) if c.richness > old.richness else (old, c)
|
||
best.domain = best.domain or other.domain
|
||
best.staging = best.staging or other.staging
|
||
best.colors = best.colors or other.colors
|
||
best.services = best.services or other.services
|
||
by_key[c.key] = best
|
||
companies = sorted(by_key.values(), key=lambda c: c.slug)
|
||
stats["kept"] = len(companies)
|
||
return companies, stats
|