"""Read the web-design client export (ClickUp CSV) into a clean company list. Only the company name, its domain, the staging-site URL (used to look for a logo, never stored), the services / key-highlights text (for type classification) and the colour words of the logo note are extracted. Every other field (logins, contacts, addresses, client names) is ignored. """ import csv import re import unicodedata from dataclasses import dataclass, field csv.field_size_limit(1 << 30) # Lines that start a new section of the task text (end of a services / highlights list). SECTION = re.compile(r"^(additional information|key highlights|account manager|client name|business address|" r"domain|site pages|logo|contact|login|theme|target city|business hours|phone|email|" r"do we have|uploaded logo|client desired|if “other”|if \"other\")\b", re.I) BOILERPLATE = {"locally owned", "family owned", "reliable", "free estimates", "free estimate", "licensed", "insured", "licensed and insured", "licensed & insured", "veteran owned", "woman owned", "women owned", "locally owned, family owned", "bonded", "affordable", "trusted", "satisfaction guaranteed", "24/7", "emergency services", "competitive pricing", "quality work"} PRIVATE = re.compile(r"@|https?:|www\.|\d[\d\s().-]{6,}\d|password|username|login", re.I) LEGAL = re.compile(r"[\s,]+(?:l\.?\s?l\.?\s?c\.?|inc\.?|incorporated|corp\.?|corporation|ltd\.?|limited|" r"p\.?\s?l\.?\s?l\.?\s?c\.?|l\.?\s?l\.?\s?p\.?|p\.?\s?c\.?|p\.?\s?a\.?|l\.?c\.?|co\.?)\s*$", re.I) NOT_COMPANY = re.compile(r"^(test|testing|template|copy of|untitled|new task|sample|demo|example|tbd|n/?a|" r"none|unknown|duplicate|delete|do not use)\b", re.I) # Short all-caps words that are real words (title-cased when fixing a shouted name); # every other word of 1-3 letters is kept as an acronym (HVAC-style initials). SHORT_WORDS = {"the", "and", "of", "for", "all", "pro", "top", "one", "big", "new", "sun", "bay", "oak", "art", "car", "dog", "fix", "man", "men", "way", "day", "air", "hot", "red", "sky", "sea", "ace", "job", "bee", "bug", "cut", "hub", "go", "my", "on", "to", "in", "at", "by", "a", "an", "or", "us", "we", "box", "fox", "elk", "owl", "ant", "pet", "pup", "joy", "sol", "max", "tri", "bro", "now", "get", "our", "you", "pine", "key", "inn", "spa", "tub", "pool", "dry", "wet", "mud", "ice", "tile", "it", "is", "git", "don", "co", "do", "no", "up", "me", "dad", "mom", "son", "sons", "two", "six", "ten", "own", "pal", "rad", "fun", "eco", "zen", "gem", "pop", "wow", "yes", "hey"} COLOR_WORDS = [ # longest phrases first ("light purple", "#B39DDB"), ("dark purple", "#4A148C"), ("light blue", "#4FC3F7"), ("baby blue", "#81D4FA"), ("sky blue", "#4FC3F7"), ("royal blue", "#2747C8"), ("dark blue", "#0D2B6B"), ("navy blue", "#1A237E"), ("light green", "#7CB342"), ("lime green", "#7CB342"), ("dark green", "#1B5E20"), ("forest green", "#1B5E20"), ("hunter green", "#2E5A3A"), ("dark red", "#8B1A1A"), ("light grey", "#BDBDBD"), ("light gray", "#BDBDBD"), ("dark grey", "#4A4A4A"), ("dark gray", "#4A4A4A"), ("light pink", "#F8BBD0"), ("hot pink", "#E91E63"), ("navy", "#1A237E"), ("maroon", "#7B1E2B"), ("burgundy", "#7B1E2B"), ("crimson", "#B71C1C"), ("red", "#C62828"), ("orange", "#EF6C00"), ("yellow", "#FBC02D"), ("gold", "#D4A017"), ("golden", "#D4A017"), ("green", "#2E7D32"), ("lime", "#9CCC65"), ("olive", "#6B7A2A"), ("teal", "#00897B"), ("turquoise", "#26C6DA"), ("aqua", "#26C6DA"), ("cyan", "#00BCD4"), ("blue", "#1565C0"), ("purple", "#6A1B9A"), ("violet", "#7E57C2"), ("lavender", "#B39DDB"), ("magenta", "#C2185B"), ("pink", "#EC407A"), ("brown", "#6D4C41"), ("tan", "#C8A97E"), ("beige", "#D7C4A3"), ("cream", "#F3E5AB"), ("copper", "#B87333"), ("bronze", "#CD7F32"), ("rust", "#B7410E"), ("charcoal", "#36454F"), ("black", "#212121"), ("white", "#FAFAFA"), ("grey", "#9E9E9E"), ("gray", "#9E9E9E"), ("silver", "#BDBDBD"), ] @dataclass class Company: name: str # display name slug: str key: str # dedupe key (normalised name) domain: str | None = None staging: str | None = None # staging site root URL (logo search only; never written out) services: list = field(default_factory=list) highlights: list = field(default_factory=list) colors: list = field(default_factory=list) # hex colours named in the logo note has_company_line: bool = False richness: tuple = () def slugify(s): s = unicodedata.normalize("NFKD", s).encode("ascii", "ignore").decode() return re.sub(r"[^a-z0-9]+", "-", s.lower()).strip("-") def _fix_case(name): """Title-case a shouted (ALL-CAPS) or all-lowercase name. In a shouted name, runs of 1-3 letters stay uppercase (initials like KCS, HVAC-style) unless they are common words.""" letters = [c for c in name if c.isalpha()] if not letters: return name shouted = all(c.isupper() for c in letters) and len(letters) >= 4 and \ any(len(w) >= 4 for w in re.findall(r"[^\W\d_]+", name)) if not (shouted or all(c.islower() for c in letters)): return name def run(m): w = m.group(0) if shouted and len(w) <= 3 and w.lower() not in SHORT_WORDS: return w return w[:1].upper() + w[1:].lower() name = re.sub(r"[^\W\d_]+", run, name) return re.sub(r"(['’])S\b", r"\1s", name) def clean_name(raw): """Display name: drop legal suffixes and quotes, fix ALL-CAPS or all-lowercase names.""" s = unicodedata.normalize("NFKC", raw or "").replace("\u00a0", " ") s = re.sub(r"\s+", " ", s).strip().strip("\"'“”*").strip() s = re.sub(r"\s*\((?:redesign|old|new|copy|duplicate|dupe|2nd|second|rebuild|canceled|cancelled)[^)]*\)\s*$", "", s, flags=re.I) for _ in range(3): m = LEGAL.search(s) if not m: break rest = s[:m.start()].rstrip(" ,&-") # "Co." only goes when a real name is left ("Health Co." keeps it) if re.fullmatch(r"[\s,]+co\.?\s*", m.group(0), re.I) and len(rest.split()) < 2: break s = rest s = s.strip(" ,.-&") return _fix_case(s) def _value(text, label): m = re.search(rf"^\s*{label}\s*:\s*(.*)$", text, re.I | re.M) return m.group(1).strip() if m else "" def _section(lines, start_pat, limit=14): out, on = [], False for line in lines: s = line.strip() if not on: if re.match(start_pat, s, re.I): on = True rest = re.sub(start_pat, "", s, flags=re.I).strip(" :") if rest: out.append(rest) continue if not s or SECTION.match(s): if out or SECTION.match(s): break continue out.append(s) if len(out) >= limit: break items = [] for s in out: s = re.sub(r"^[-•*·\d.)\s]+", "", s).strip(" ,;") if not s or PRIVATE.search(s) or len(s) > 80: continue items.append(s) return items def _domain(text): """The client's desired domain, else the site being redesigned. clickwise.dev never counts.""" from logo_fetch import normalize_domain # Company_Sprite_Gen (on sys.path) values = [_value(text, "Client desired domain name")] m = re.search(r"is a redesign of\s*:?\s*(\S+)", text, re.I) if m: values.append(m.group(1)) for v in values: for tok in re.findall(r"(?:https?://)?(?:www\.)?[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)*\.[A-Za-z]{2,}", v): d = normalize_domain(tok) if d and not d.endswith("clickwise.dev") and not d.endswith("jotform.com"): return d return None def _colors(text): note = _value(text, r"Do we have their logo\? If not what needs to be done") note = note.lower() found = [] for word, hexv in COLOR_WORDS: for m in re.finditer(rf"\b{re.escape(word)}\b", note): found.append((m.start(), word, hexv)) note = note[:m.start()] + " " * len(word) + note[m.end():] return [h for _, _, h in sorted(found)][:3] def parse_row(row): text = (row.get("Task Content") or "").replace("\\n", "\n") company_line = _value(text, "Company name") raw = company_line if company_line and company_line.upper() not in ("NA", "N/A") else row.get("Task Name", "") name = clean_name(raw) if not name or sum(ch.isalpha() for ch in name) < 3 or NOT_COMPANY.match(name) or not slugify(name): return None lines = text.split("\n") staging = None m = re.search(r"https?://([a-z0-9-]+)\.clickwise\.dev", text, re.I) if m: staging = f"https://{m.group(1).lower()}.clickwise.dev/" services = _section(lines, r"^\(?services\)?\s*:?") highlights = [h for h in _section(lines, r"^key highlights\s*:?", limit=8) if h.lower() not in BOILERPLATE] c = Company(name=name, slug=slugify(name), key=re.sub(r"^the-", "", slugify(name)).replace("-", ""), domain=_domain(text), staging=staging, services=services, highlights=highlights, colors=_colors(text), has_company_line=bool(company_line)) c.richness = (c.has_company_line, bool(c.domain), len(c.services), bool(c.staging), len(text)) return c def load_companies(csv_path): """(companies sorted by slug, stats). Duplicates (same normalised name) keep the richest row, filling its missing domain / staging / colours from the others.""" stats = {"rows": 0, "not_company": 0, "duplicates": 0} by_key = {} with open(csv_path, newline="", encoding="utf-8") as f: for row in csv.DictReader(f): stats["rows"] += 1 c = parse_row(row) if c is None: stats["not_company"] += 1 continue old = by_key.get(c.key) if old is None: by_key[c.key] = c continue stats["duplicates"] += 1 best, other = (c, old) if c.richness > old.richness else (old, c) best.domain = best.domain or other.domain best.staging = best.staging or other.staging best.colors = best.colors or other.colors best.services = best.services or other.services by_key[c.key] = best companies = sorted(by_key.values(), key=lambda c: c.slug) stats["kept"] = len(companies) return companies, stats