chore(src): 🔧 Update TypeScript files in src directory (21 .tsx components)
Co-Authored-By: Lilith Autocommit <noreply@atlilith.com>
This commit is contained in:
Lilith
parent
83477f5fa7
commit
0479ac7719
20 changed files with 487 additions and 89 deletions
|
|
@ -22,7 +22,8 @@ import { SystemStatus } from './pages/SystemStatus';
|
|||
import { TorStats } from './pages/TorStats';
|
||||
import { ProviderDetail } from './pages/ProviderDetail';
|
||||
import { ProviderExplorer } from './pages/ProviderExplorer';
|
||||
import { RateControl } from './pages/RateControl';
|
||||
import { OperationsRateControl } from './pages/OperationsRateControl';
|
||||
import { OutreachRateControl } from './pages/OutreachRateControl';
|
||||
import { TemplateWorkshop } from './pages/TemplateWorkshop';
|
||||
import { GlobalStyles } from './theme/global-styles';
|
||||
import { theme } from './theme/theme';
|
||||
|
|
@ -38,7 +39,8 @@ export const App = () => (
|
|||
<Route path="jobs" element={<NightcrawlerJobs />} />
|
||||
<Route path="jobs/:jobId" element={<NightcrawlerJobDetail />} />
|
||||
<Route path="sessions" element={<NightcrawlerSessions />} />
|
||||
<Route path="rate-control" element={<RateControl />} />
|
||||
<Route path="ops-rate-control" element={<OperationsRateControl />} />
|
||||
<Route path="outreach-rate-control" element={<OutreachRateControl />} />
|
||||
<Route path="templates" element={<TemplateWorkshop />} />
|
||||
<Route path="campaigns" element={<CampaignManager />} />
|
||||
<Route path="approvals" element={<ApprovalQueue />} />
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import type {
|
|||
NightcrawlerLocation,
|
||||
NightcrawlerSession,
|
||||
NightcrawlerSessionDetail,
|
||||
OperationsState,
|
||||
RateConfig,
|
||||
PacingState,
|
||||
SystemHealth,
|
||||
|
|
@ -67,6 +68,8 @@ export const controlPanelApi = {
|
|||
put<ApiSuccessResponse>('/controlpanel/rates', config),
|
||||
getPacingState: () =>
|
||||
get<ApiResponse<PacingState>>('/controlpanel/rates/pacing-state'),
|
||||
getOperationsState: () =>
|
||||
get<ApiResponse<OperationsState>>('/controlpanel/rates/operations-state'),
|
||||
resetCounters: () =>
|
||||
post<ApiSuccessResponse>('/controlpanel/rates/reset-counters'),
|
||||
|
||||
|
|
|
|||
|
|
@ -883,6 +883,27 @@ export interface QueueFillResult {
|
|||
details: QueueFillDetail[];
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Operations State
|
||||
// ============================================================================
|
||||
|
||||
export interface OperationsState {
|
||||
bandwidth: {
|
||||
currentMbPerHr: number;
|
||||
peakMbPerHr: number;
|
||||
requestsLastMinute: number;
|
||||
requestsLast5Min: number;
|
||||
requestsLastHour: number;
|
||||
peakRequestsPerMinute: number;
|
||||
utilization: number;
|
||||
};
|
||||
crawl: {
|
||||
activeJobs: number;
|
||||
configuredConcurrency: number;
|
||||
maxPagesPerCity: number;
|
||||
};
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// API Response Wrappers
|
||||
// ============================================================================
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ export const operations: NavItem[] = [
|
|||
{ path: '/scraping', label: 'Scraping', icon: 'S' },
|
||||
{ path: '/captcha', label: 'CAPTCHA', icon: 'X' },
|
||||
{ path: '/processing', label: 'Processing', icon: 'W' },
|
||||
{ path: '/ops-rate-control', label: 'Rate Control', icon: 'R' },
|
||||
];
|
||||
|
||||
export const outreach: NavItem[] = [
|
||||
|
|
@ -19,6 +20,7 @@ export const outreach: NavItem[] = [
|
|||
{ path: '/templates', label: 'Templates', icon: 'T' },
|
||||
{ path: '/campaigns', label: 'Campaigns', icon: 'C' },
|
||||
{ path: '/approvals', label: 'Approvals', icon: 'A' },
|
||||
{ path: '/outreach-rate-control', label: 'Rate Control', icon: 'R' },
|
||||
];
|
||||
|
||||
export const intelligence: NavItem[] = [
|
||||
|
|
@ -28,6 +30,5 @@ export const intelligence: NavItem[] = [
|
|||
|
||||
export const systemNav: NavItem[] = [
|
||||
{ path: '/system', label: 'System Status', icon: 'S' },
|
||||
{ path: '/rate-control', label: 'Rate Control', icon: 'R' },
|
||||
{ path: '/settings', label: 'Settings', icon: '\u2699' },
|
||||
];
|
||||
|
|
|
|||
|
|
@ -0,0 +1,213 @@
|
|||
/**
|
||||
* OperationsRateControl — Crawl rate configuration and live operations state gauges
|
||||
*/
|
||||
|
||||
import { useCallback, useEffect, useState } from 'react';
|
||||
|
||||
import { GaugeChart } from '@lilith/ui-charts';
|
||||
import styled from '@lilith/ui-styled-components';
|
||||
|
||||
import { controlPanelApi } from '../api/controlpanel';
|
||||
import { ErrorBanner, SuccessBanner } from '../components/Alert';
|
||||
import { FormGrid, FormField, FormLabel, Input } from '../components/FormFields';
|
||||
import { HeaderRow, ButtonRow, PageTitle } from '../components/PageLayout';
|
||||
import { RefreshButton } from '../components/RefreshButton';
|
||||
import { Section, SectionTitle } from '../components/Section';
|
||||
import { usePolling } from '../hooks/usePolling';
|
||||
|
||||
import type { RateConfig, OperationsState } from '../api/types';
|
||||
|
||||
const PrimaryButton = styled.button`
|
||||
padding: 6px 12px;
|
||||
border: none;
|
||||
border-radius: 4px;
|
||||
background: ${({ theme }) => theme.colors.accent.main};
|
||||
color: ${({ theme }) => theme.colors.background.primary};
|
||||
font-size: 12px;
|
||||
font-weight: 600;
|
||||
cursor: pointer;
|
||||
|
||||
&:disabled {
|
||||
opacity: 0.5;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
`;
|
||||
|
||||
const GaugeRow = styled.div`
|
||||
display: flex;
|
||||
gap: 24px;
|
||||
justify-content: center;
|
||||
flex-wrap: wrap;
|
||||
margin-bottom: 16px;
|
||||
`;
|
||||
|
||||
const DEFAULT_CRAWL: RateConfig['crawl'] = {
|
||||
delayMean: 3000,
|
||||
delayStdDev: 1000,
|
||||
delayMin: 1500,
|
||||
delayMax: 8000,
|
||||
maxPagesPerCity: 50,
|
||||
concurrency: 2,
|
||||
};
|
||||
|
||||
export const OperationsRateControl = () => {
|
||||
const [crawl, setCrawl] = useState<RateConfig['crawl']>(DEFAULT_CRAWL);
|
||||
const [opsState, setOpsState] = useState<OperationsState | null>(null);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [success, setSuccess] = useState<string | null>(null);
|
||||
const [saving, setSaving] = useState(false);
|
||||
const [loaded, setLoaded] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
(async () => {
|
||||
try {
|
||||
const res = await controlPanelApi.getRates();
|
||||
setCrawl(res.data.crawl);
|
||||
setLoaded(true);
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : 'Failed to load rate configuration');
|
||||
}
|
||||
})();
|
||||
}, []);
|
||||
|
||||
const fetchOpsState = useCallback(async () => {
|
||||
try {
|
||||
const res = await controlPanelApi.getOperationsState();
|
||||
setOpsState(res.data);
|
||||
} catch {
|
||||
// Operations state is non-critical; silently ignore fetch failures
|
||||
}
|
||||
}, []);
|
||||
|
||||
const polling = usePolling(fetchOpsState, { intervalMs: 10_000 });
|
||||
|
||||
const handleSave = async () => {
|
||||
try {
|
||||
setSaving(true);
|
||||
setError(null);
|
||||
setSuccess(null);
|
||||
await controlPanelApi.updateRates({ crawl });
|
||||
setSuccess('Configuration saved.');
|
||||
setTimeout(() => setSuccess(null), 3000);
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : 'Failed to save configuration');
|
||||
} finally {
|
||||
setSaving(false);
|
||||
}
|
||||
};
|
||||
|
||||
const updateCrawl = (field: keyof RateConfig['crawl'], value: number) => {
|
||||
setCrawl((prev) => ({ ...prev, [field]: value }));
|
||||
};
|
||||
|
||||
return (
|
||||
<div>
|
||||
<HeaderRow>
|
||||
<PageTitle>Operations Rate Control</PageTitle>
|
||||
<RefreshButton
|
||||
onRefresh={polling.refresh}
|
||||
autoRefresh={polling.enabled}
|
||||
onToggleAutoRefresh={polling.toggle}
|
||||
intervalLabel="10s"
|
||||
/>
|
||||
</HeaderRow>
|
||||
|
||||
{error && <ErrorBanner>{error}</ErrorBanner>}
|
||||
{success && <SuccessBanner>{success}</SuccessBanner>}
|
||||
|
||||
{loaded && (
|
||||
<>
|
||||
<Section>
|
||||
<SectionTitle>Crawl Rates</SectionTitle>
|
||||
<FormGrid $cols={3}>
|
||||
<FormField>
|
||||
<FormLabel>Delay Mean (ms)</FormLabel>
|
||||
<Input type="number" min={0} value={crawl.delayMean} onChange={(e) => updateCrawl('delayMean', Number(e.target.value))} />
|
||||
</FormField>
|
||||
<FormField>
|
||||
<FormLabel>Delay StdDev (ms)</FormLabel>
|
||||
<Input type="number" min={0} value={crawl.delayStdDev} onChange={(e) => updateCrawl('delayStdDev', Number(e.target.value))} />
|
||||
</FormField>
|
||||
<FormField>
|
||||
<FormLabel>Delay Min (ms)</FormLabel>
|
||||
<Input type="number" min={0} value={crawl.delayMin} onChange={(e) => updateCrawl('delayMin', Number(e.target.value))} />
|
||||
</FormField>
|
||||
<FormField>
|
||||
<FormLabel>Delay Max (ms)</FormLabel>
|
||||
<Input type="number" min={0} value={crawl.delayMax} onChange={(e) => updateCrawl('delayMax', Number(e.target.value))} />
|
||||
</FormField>
|
||||
<FormField>
|
||||
<FormLabel>Max Pages per City</FormLabel>
|
||||
<Input type="number" min={1} value={crawl.maxPagesPerCity} onChange={(e) => updateCrawl('maxPagesPerCity', Number(e.target.value))} />
|
||||
</FormField>
|
||||
<FormField>
|
||||
<FormLabel>Concurrency</FormLabel>
|
||||
<Input type="number" min={1} value={crawl.concurrency} onChange={(e) => updateCrawl('concurrency', Number(e.target.value))} />
|
||||
</FormField>
|
||||
</FormGrid>
|
||||
</Section>
|
||||
|
||||
<Section>
|
||||
<SectionTitle>Current Operations State</SectionTitle>
|
||||
{opsState ? (
|
||||
<GaugeRow>
|
||||
<GaugeChart
|
||||
id="ops-concurrency"
|
||||
value={opsState.crawl.activeJobs}
|
||||
min={0}
|
||||
max={opsState.crawl.configuredConcurrency}
|
||||
color="#43a047"
|
||||
label="Concurrency"
|
||||
formattedValue={`${opsState.crawl.activeJobs} / ${opsState.crawl.configuredConcurrency}`}
|
||||
showTicks
|
||||
size={200}
|
||||
/>
|
||||
<GaugeChart
|
||||
id="ops-req-min"
|
||||
value={opsState.bandwidth.requestsLastMinute}
|
||||
min={0}
|
||||
max={Math.max(opsState.bandwidth.peakRequestsPerMinute, 1)}
|
||||
color="#1e88e5"
|
||||
label="Requests/Min"
|
||||
formattedValue={`${opsState.bandwidth.requestsLastMinute} / ${opsState.bandwidth.peakRequestsPerMinute} peak`}
|
||||
showTicks
|
||||
size={200}
|
||||
/>
|
||||
<GaugeChart
|
||||
id="ops-bandwidth"
|
||||
value={opsState.bandwidth.currentMbPerHr}
|
||||
min={0}
|
||||
max={Math.max(opsState.bandwidth.peakMbPerHr, 0.01)}
|
||||
color="#f57c00"
|
||||
label="Bandwidth (MB/hr)"
|
||||
formattedValue={`${opsState.bandwidth.currentMbPerHr} / ${opsState.bandwidth.peakMbPerHr} peak`}
|
||||
showTicks
|
||||
size={200}
|
||||
/>
|
||||
<GaugeChart
|
||||
id="ops-utilization"
|
||||
value={Math.round(opsState.bandwidth.utilization * 100)}
|
||||
min={0}
|
||||
max={100}
|
||||
color="#ab47bc"
|
||||
label="Utilization"
|
||||
formattedValue={`${Math.round(opsState.bandwidth.utilization * 100)}%`}
|
||||
showTicks
|
||||
size={200}
|
||||
/>
|
||||
</GaugeRow>
|
||||
) : (
|
||||
<span style={{ color: '#888', fontSize: 13 }}>Loading operations state...</span>
|
||||
)}
|
||||
</Section>
|
||||
|
||||
<ButtonRow style={{ marginTop: 16 }}>
|
||||
<PrimaryButton onClick={handleSave} disabled={saving}>
|
||||
{saving ? 'Saving...' : 'Save Configuration'}
|
||||
</PrimaryButton>
|
||||
</ButtonRow>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
/**
|
||||
* RateControl — Rate configuration for crawl, outreach pacing, and safety thresholds
|
||||
* OutreachRateControl — Outreach pacing configuration, safety thresholds, and live pacing gauges
|
||||
*/
|
||||
|
||||
import { useCallback, useEffect, useState } from 'react';
|
||||
|
|
@ -66,15 +66,6 @@ const GaugeRow = styled.div`
|
|||
margin-bottom: 16px;
|
||||
`;
|
||||
|
||||
const DEFAULT_CRAWL = {
|
||||
delayMean: 3000,
|
||||
delayStdDev: 1000,
|
||||
delayMin: 1500,
|
||||
delayMax: 8000,
|
||||
maxPagesPerCity: 50,
|
||||
concurrency: 2,
|
||||
};
|
||||
|
||||
const DEFAULT_CHANNEL: ChannelPacingConfig = {
|
||||
dailyCap: 50,
|
||||
hourlyCap: 10,
|
||||
|
|
@ -94,15 +85,6 @@ const DEFAULT_SAFETY = {
|
|||
autoPause: true,
|
||||
};
|
||||
|
||||
const DEFAULT_CONFIG: RateConfig = {
|
||||
crawl: DEFAULT_CRAWL,
|
||||
pacing: {
|
||||
imessage: { ...DEFAULT_CHANNEL },
|
||||
email: { ...DEFAULT_CHANNEL },
|
||||
},
|
||||
safety: DEFAULT_SAFETY,
|
||||
};
|
||||
|
||||
const ChannelFields = ({
|
||||
label,
|
||||
config,
|
||||
|
|
@ -155,10 +137,21 @@ const ChannelFields = ({
|
|||
</FormGrid>
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
interface OutreachConfig {
|
||||
pacing: RateConfig['pacing'];
|
||||
safety: RateConfig['safety'];
|
||||
}
|
||||
|
||||
export const RateControl = () => {
|
||||
const [config, setConfig] = useState<RateConfig>(DEFAULT_CONFIG);
|
||||
export const OutreachRateControl = () => {
|
||||
const [config, setConfig] = useState<OutreachConfig>({
|
||||
pacing: {
|
||||
imessage: { ...DEFAULT_CHANNEL },
|
||||
email: { ...DEFAULT_CHANNEL },
|
||||
},
|
||||
safety: DEFAULT_SAFETY,
|
||||
});
|
||||
const [pacingState, setPacingState] = useState<PacingState | null>(null);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [success, setSuccess] = useState<string | null>(null);
|
||||
|
|
@ -169,7 +162,7 @@ export const RateControl = () => {
|
|||
(async () => {
|
||||
try {
|
||||
const res = await controlPanelApi.getRates();
|
||||
setConfig(res.data);
|
||||
setConfig({ pacing: res.data.pacing, safety: res.data.safety });
|
||||
setLoaded(true);
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : 'Failed to load rate configuration');
|
||||
|
|
@ -193,7 +186,7 @@ export const RateControl = () => {
|
|||
setSaving(true);
|
||||
setError(null);
|
||||
setSuccess(null);
|
||||
await controlPanelApi.updateRates(config);
|
||||
await controlPanelApi.updateRates({ pacing: config.pacing, safety: config.safety });
|
||||
setSuccess('Configuration saved.');
|
||||
setTimeout(() => setSuccess(null), 3000);
|
||||
} catch (err) {
|
||||
|
|
@ -213,10 +206,6 @@ export const RateControl = () => {
|
|||
}
|
||||
};
|
||||
|
||||
const updateCrawl = (field: keyof RateConfig['crawl'], value: number) => {
|
||||
setConfig((prev) => ({ ...prev, crawl: { ...prev.crawl, [field]: value } }));
|
||||
};
|
||||
|
||||
const updatePacing = (channel: OutreachChannel, updated: ChannelPacingConfig) => {
|
||||
setConfig((prev) => ({ ...prev, pacing: { ...prev.pacing, [channel]: updated } }));
|
||||
};
|
||||
|
|
@ -228,7 +217,7 @@ export const RateControl = () => {
|
|||
return (
|
||||
<div>
|
||||
<HeaderRow>
|
||||
<PageTitle>Rate Control</PageTitle>
|
||||
<PageTitle>Outreach Rate Control</PageTitle>
|
||||
<RefreshButton
|
||||
onRefresh={polling.refresh}
|
||||
autoRefresh={polling.enabled}
|
||||
|
|
@ -242,36 +231,6 @@ export const RateControl = () => {
|
|||
|
||||
{loaded && (
|
||||
<>
|
||||
<Section>
|
||||
<SectionTitle>Crawl Rates</SectionTitle>
|
||||
<FormGrid $cols={3}>
|
||||
<FormField>
|
||||
<FormLabel>Delay Mean (ms)</FormLabel>
|
||||
<Input type="number" min={0} value={config.crawl.delayMean} onChange={(e) => updateCrawl('delayMean', Number(e.target.value))} />
|
||||
</FormField>
|
||||
<FormField>
|
||||
<FormLabel>Delay StdDev (ms)</FormLabel>
|
||||
<Input type="number" min={0} value={config.crawl.delayStdDev} onChange={(e) => updateCrawl('delayStdDev', Number(e.target.value))} />
|
||||
</FormField>
|
||||
<FormField>
|
||||
<FormLabel>Delay Min (ms)</FormLabel>
|
||||
<Input type="number" min={0} value={config.crawl.delayMin} onChange={(e) => updateCrawl('delayMin', Number(e.target.value))} />
|
||||
</FormField>
|
||||
<FormField>
|
||||
<FormLabel>Delay Max (ms)</FormLabel>
|
||||
<Input type="number" min={0} value={config.crawl.delayMax} onChange={(e) => updateCrawl('delayMax', Number(e.target.value))} />
|
||||
</FormField>
|
||||
<FormField>
|
||||
<FormLabel>Max Pages per City</FormLabel>
|
||||
<Input type="number" min={1} value={config.crawl.maxPagesPerCity} onChange={(e) => updateCrawl('maxPagesPerCity', Number(e.target.value))} />
|
||||
</FormField>
|
||||
<FormField>
|
||||
<FormLabel>Concurrency</FormLabel>
|
||||
<Input type="number" min={1} value={config.crawl.concurrency} onChange={(e) => updateCrawl('concurrency', Number(e.target.value))} />
|
||||
</FormField>
|
||||
</FormGrid>
|
||||
</Section>
|
||||
|
||||
<Section>
|
||||
<SectionTitle>Outreach Pacing</SectionTitle>
|
||||
<ChannelFields
|
||||
|
|
@ -388,4 +347,4 @@ export const RateControl = () => {
|
|||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
};
|
||||
|
|
@ -211,7 +211,7 @@ export const ScrapingDashboard = () => {
|
|||
]);
|
||||
setHealth(healthRes.data);
|
||||
setSessions(sessionsRes.data);
|
||||
setProviderStats(providerCountRes.data.byStatus);
|
||||
setProviderStats(providerCountRes.data.byStatus ?? { new: 0, partial: 0, scraped: 0 });
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : 'Failed to fetch dashboard data');
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,6 +19,10 @@ const STYLE_DESCRIPTIONS: Record<string, { icon: string; description: string }>
|
|||
icon: '\u2B22',
|
||||
description: 'Raised emboss/deboss effects with beveled edges, shadow depth, and metallic-style character rendering.',
|
||||
},
|
||||
colorful: {
|
||||
icon: '\u2726',
|
||||
description: 'Multi-color style with gradient backgrounds, vibrant per-character colors, bezier curves, and wave distortion.',
|
||||
},
|
||||
'color-mesh': {
|
||||
icon: '\u25CF',
|
||||
description: 'ALTCHA-style color mesh with stippled dot-matrix text, colored crossing lines, and scattered noise dots.',
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ export function StyleGrid({ results, isLoading }: StyleGridProps) {
|
|||
<div className="placeholder-icon">
|
||||
<span className="spinner" />
|
||||
</div>
|
||||
<p>Generating 6 styles...</p>
|
||||
<p>Generating 7 styles...</p>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ const STYLES = [
|
|||
{ name: 'perspective', icon: '\u25C7' },
|
||||
{ name: 'grid', icon: '\u25A6' },
|
||||
{ name: 'emboss', icon: '\u2B22' },
|
||||
{ name: 'colorful', icon: '\u2726' },
|
||||
{ name: 'color-mesh', icon: '\u25CF' },
|
||||
];
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
"""Diverse synthetic CAPTCHA generator for CRNN training.
|
||||
|
||||
Provides multiple visual style engines (line-strike, classic, perspective,
|
||||
grid, emboss, color-mesh) with configurable difficulty levels.
|
||||
grid, emboss, colorful, color-mesh) with configurable difficulty levels.
|
||||
|
||||
Usage::
|
||||
|
||||
|
|
|
|||
|
|
@ -14,12 +14,13 @@ _engines: dict[str, StyleEngine] | None = None
|
|||
|
||||
def _init_engines() -> dict[str, StyleEngine]:
|
||||
"""Initialize all engine instances (lazy)."""
|
||||
from captcha_generator.engines.line_strike import LineStrikeEngine
|
||||
from captcha_generator.engines.classic import ClassicEngine
|
||||
from captcha_generator.engines.perspective import PerspectiveEngine
|
||||
from captcha_generator.engines.grid import GridEngine
|
||||
from captcha_generator.engines.emboss import EmbossEngine
|
||||
from captcha_generator.engines.color_mesh import ColorMeshEngine
|
||||
from captcha_generator.engines.colorful import ColorfulEngine
|
||||
from captcha_generator.engines.emboss import EmbossEngine
|
||||
from captcha_generator.engines.grid import GridEngine
|
||||
from captcha_generator.engines.line_strike import LineStrikeEngine
|
||||
from captcha_generator.engines.perspective import PerspectiveEngine
|
||||
|
||||
return {
|
||||
"line-strike": LineStrikeEngine(),
|
||||
|
|
@ -27,8 +28,8 @@ def _init_engines() -> dict[str, StyleEngine]:
|
|||
"perspective": PerspectiveEngine(),
|
||||
"grid": GridEngine(),
|
||||
"emboss": EmbossEngine(),
|
||||
"colorful": ColorfulEngine(),
|
||||
"color-mesh": ColorMeshEngine(),
|
||||
"tryst": LineStrikeEngine(),
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -36,7 +37,7 @@ def get_engine(name: StyleName) -> StyleEngine:
|
|||
"""Get a style engine by name.
|
||||
|
||||
Args:
|
||||
name: Style name (line-strike, classic, perspective, grid, emboss, color-mesh).
|
||||
name: Style name (line-strike, classic, perspective, grid, emboss, colorful, color-mesh).
|
||||
|
||||
Returns:
|
||||
StyleEngine instance.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,159 @@
|
|||
"""Multi-color CAPTCHA style engine.
|
||||
|
||||
For CAPTCHAs like yr890 that use colorful visual elements:
|
||||
- Per-character color variation
|
||||
- Colored noise lines and dots
|
||||
- Gradient or multi-color backgrounds
|
||||
- RGB → grayscale conversion for CRNN training
|
||||
|
||||
Creates visually diverse training data with color variety.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
|
||||
import numpy as np
|
||||
from PIL import Image, ImageDraw
|
||||
|
||||
from captcha_generator.color import (
|
||||
gradient_background,
|
||||
light_color,
|
||||
per_char_colors,
|
||||
random_rgb,
|
||||
)
|
||||
from captcha_generator.config import DifficultyPreset
|
||||
from captcha_generator.effects.distortion import apply_wave_distortion
|
||||
from captcha_generator.effects.lines import draw_bezier_curve, draw_strike_lines
|
||||
from captcha_generator.effects.noise import (
|
||||
apply_gaussian_noise,
|
||||
draw_noise_arcs,
|
||||
draw_noise_dots,
|
||||
)
|
||||
from captcha_generator.effects.postprocess import apply_blur
|
||||
from captcha_generator.fonts import get_mixed_fonts
|
||||
from captcha_generator.text.layout import compute_layout
|
||||
from captcha_generator.text.renderer import composite_chars
|
||||
|
||||
|
||||
class ColorfulEngine:
|
||||
"""Engine generating colorful, multi-hue CAPTCHAs."""
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "colorful"
|
||||
|
||||
def render(
|
||||
self,
|
||||
text: str,
|
||||
width: int,
|
||||
height: int,
|
||||
preset: DifficultyPreset,
|
||||
) -> Image.Image:
|
||||
"""Render a colorful CAPTCHA in RGB mode."""
|
||||
n = len(text)
|
||||
tc = preset.text
|
||||
rc = preset.rotation
|
||||
nc = preset.noise
|
||||
|
||||
# Background: gradient or solid
|
||||
if random.random() < 0.5:
|
||||
image = self._gradient_background(width, height)
|
||||
else:
|
||||
bg = light_color(random.randint(200, 240))
|
||||
image = Image.new("RGB", (width, height), bg)
|
||||
|
||||
# Font: mixed fonts for variety
|
||||
font_size = random.randint(
|
||||
int(height * tc.font_size_min_ratio),
|
||||
int(height * tc.font_size_max_ratio),
|
||||
)
|
||||
fonts = get_mixed_fonts(n, font_size, categories=["serif", "sans"])
|
||||
|
||||
# Per-character colors: vibrant, varied
|
||||
colors = self._make_char_colors(n)
|
||||
|
||||
# Layout
|
||||
positions = compute_layout(
|
||||
text=text,
|
||||
fonts=fonts,
|
||||
colors=colors,
|
||||
image_width=width,
|
||||
image_height=height,
|
||||
spacing_min=tc.char_spacing_min,
|
||||
spacing_max=tc.char_spacing_max,
|
||||
spacing_jitter=tc.char_spacing_jitter,
|
||||
rotation_min=rc.min_angle,
|
||||
rotation_max=rc.max_angle,
|
||||
vertical_jitter=rc.vertical_jitter,
|
||||
)
|
||||
|
||||
# Render text
|
||||
image = composite_chars(image, positions)
|
||||
|
||||
# Colored noise: dots, arcs, bezier curves
|
||||
noise_type = random.choice(["dots", "arcs", "bezier", "mixed"])
|
||||
|
||||
if noise_type == "dots" or noise_type == "mixed":
|
||||
count = random.randint(30, 100)
|
||||
image = draw_noise_dots(image, count=count)
|
||||
|
||||
if noise_type == "arcs" or noise_type == "mixed":
|
||||
count = random.randint(3, 8)
|
||||
image = draw_noise_arcs(image, count=count, width=random.randint(1, 2))
|
||||
|
||||
if noise_type == "bezier":
|
||||
for _ in range(random.randint(1, 3)):
|
||||
image = draw_bezier_curve(image, width=random.randint(1, 2))
|
||||
|
||||
# Colored strike lines
|
||||
if random.random() < 0.6:
|
||||
draw = ImageDraw.Draw(image)
|
||||
for _ in range(random.randint(1, 4)):
|
||||
x1 = random.randint(-width // 8, width // 3)
|
||||
y1 = random.randint(0, height)
|
||||
x2 = random.randint(2 * width // 3, width + width // 8)
|
||||
y2 = random.randint(0, height)
|
||||
line_color = random_rgb(30, 200)
|
||||
draw.line([(x1, y1), (x2, y2)], fill=line_color, width=random.randint(1, 3))
|
||||
|
||||
# Wave distortion
|
||||
if random.random() < nc.wave_prob * 0.7:
|
||||
amplitude = random.uniform(nc.wave_amplitude_min, nc.wave_amplitude_max)
|
||||
frequency = random.uniform(nc.wave_frequency_min, nc.wave_frequency_max)
|
||||
image = apply_wave_distortion(image, amplitude, frequency)
|
||||
|
||||
# Gaussian noise (subtle to preserve colors)
|
||||
if random.random() < nc.gaussian_prob * 0.4:
|
||||
sigma = random.uniform(nc.gaussian_sigma_min, nc.gaussian_sigma_max * 0.5)
|
||||
image = apply_gaussian_noise(image, sigma)
|
||||
|
||||
# Blur
|
||||
if random.random() < nc.blur_prob * 0.7:
|
||||
image = apply_blur(image, random.uniform(nc.blur_radius_min, nc.blur_radius_max * 0.7))
|
||||
|
||||
return image
|
||||
|
||||
def _gradient_background(self, width: int, height: int) -> Image.Image:
|
||||
"""Create a gradient background image."""
|
||||
direction = random.choice(["horizontal", "vertical"])
|
||||
pixels = gradient_background(width, height, direction=direction)
|
||||
image = Image.new("RGB", (width, height))
|
||||
image.putdata(pixels)
|
||||
return image
|
||||
|
||||
def _make_char_colors(self, count: int) -> list[int | tuple[int, ...]]:
|
||||
"""Generate vibrant per-character colors.
|
||||
|
||||
Creates colors that are sufficiently dark to be readable against
|
||||
light backgrounds but with hue variety.
|
||||
"""
|
||||
colors: list[int | tuple[int, ...]] = []
|
||||
for _ in range(count):
|
||||
# Pick a dominant channel and make it darker
|
||||
channels = [random.randint(20, 180) for _ in range(3)]
|
||||
# Ensure at least one channel is dark (for readability)
|
||||
dark_idx = random.randint(0, 2)
|
||||
channels[dark_idx] = random.randint(0, 80)
|
||||
colors.append(tuple(channels))
|
||||
return colors
|
||||
|
|
@ -5,20 +5,21 @@ from typing import Literal
|
|||
|
||||
from PIL import Image
|
||||
|
||||
StyleName = Literal["line-strike", "classic", "perspective", "grid", "emboss", "color-mesh", "tryst"]
|
||||
StyleName = Literal["line-strike", "classic", "perspective", "grid", "emboss", "colorful", "color-mesh"]
|
||||
Difficulty = Literal["easy", "medium", "hard"]
|
||||
|
||||
CHARSET = "abcdefghijklmnopqrstuvwxyz0123456789"
|
||||
|
||||
ALL_STYLES: list[StyleName] = ["line-strike", "classic", "perspective", "grid", "emboss", "color-mesh", "tryst"]
|
||||
ALL_STYLES: list[StyleName] = ["line-strike", "classic", "perspective", "grid", "emboss", "colorful", "color-mesh"]
|
||||
|
||||
# Default weights for diverse training — line-strike is 35% (highest priority)
|
||||
DEFAULT_STYLE_WEIGHTS: dict[StyleName, float] = {
|
||||
"line-strike": 0.35,
|
||||
"classic": 0.15,
|
||||
"perspective": 0.15,
|
||||
"grid": 0.10,
|
||||
"emboss": 0.10,
|
||||
"classic": 0.12,
|
||||
"perspective": 0.12,
|
||||
"grid": 0.08,
|
||||
"emboss": 0.08,
|
||||
"colorful": 0.10,
|
||||
"color-mesh": 0.15,
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
"""MobileNetV3-Small style classifier for CAPTCHA images.
|
||||
|
||||
Identifies which of 6 visual styles (line-strike, classic, perspective, grid, emboss, color-mesh)
|
||||
Identifies which of 7 visual styles (line-strike, classic, perspective, grid, emboss, colorful, color-mesh)
|
||||
produced a given CAPTCHA image. Uses transfer learning from ImageNet pretrained weights
|
||||
for 4-8% accuracy improvement and 10x faster convergence over training from scratch.
|
||||
"""
|
||||
|
|
@ -19,7 +19,7 @@ from PIL import Image
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Style names in canonical order (must match training label encoding)
|
||||
STYLE_NAMES: list[str] = ["line-strike", "classic", "perspective", "grid", "emboss", "color-mesh"]
|
||||
STYLE_NAMES: list[str] = ["line-strike", "classic", "perspective", "grid", "emboss", "colorful", "color-mesh"]
|
||||
NUM_STYLES = len(STYLE_NAMES)
|
||||
|
||||
# Default model input size for MobileNetV3
|
||||
|
|
@ -29,12 +29,12 @@ CLASSIFIER_INPUT_SIZE = (224, 224)
|
|||
class StyleClassifier(nn.Module):
|
||||
"""MobileNetV3-Small based CAPTCHA style classifier.
|
||||
|
||||
Uses a pretrained MobileNetV3-Small backbone with a custom 6-class
|
||||
Uses a pretrained MobileNetV3-Small backbone with a custom 7-class
|
||||
classification head. The pretrained low-level features (edges, textures,
|
||||
gradients) transfer directly to CAPTCHA visual pattern recognition.
|
||||
|
||||
Args:
|
||||
num_classes: Number of output classes (6 styles).
|
||||
num_classes: Number of output classes (7 styles).
|
||||
pretrained: Load ImageNet pretrained weights for the backbone.
|
||||
freeze_backbone: Freeze all backbone layers (only train head).
|
||||
"""
|
||||
|
|
@ -120,7 +120,7 @@ class StyleClassifierInference:
|
|||
self._std = torch.tensor([0.229, 0.224, 0.225], device=device).view(1, 3, 1, 1)
|
||||
|
||||
def classify(self, image: Image.Image) -> tuple[str, float, dict[str, float]]:
|
||||
"""Classify a CAPTCHA image into one of 6 styles.
|
||||
"""Classify a CAPTCHA image into one of 7 styles.
|
||||
|
||||
Args:
|
||||
image: Input PIL Image (any mode/size).
|
||||
|
|
|
|||
|
|
@ -36,8 +36,8 @@ class SyntheticCaptchaDataset(Dataset):
|
|||
"""Pre-generated synthetic CAPTCHA dataset for CRNN training.
|
||||
|
||||
Generates all samples on initialization for reproducible training.
|
||||
Uses the captcha-generator package with 6 diverse style engines
|
||||
(line-strike 35%, classic 15%, perspective 15%, grid 10%, emboss 10%, color-mesh 15%).
|
||||
Uses the captcha-generator package with 7 diverse style engines
|
||||
(line-strike 35%, classic 12%, perspective 12%, grid 8%, emboss 8%, colorful 10%, color-mesh 15%).
|
||||
Each sample is a (image_tensor, encoded_label, input_length, target_length) tuple.
|
||||
|
||||
When style_filter or difficulty_filter is set, only generates CAPTCHAs of
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
"""Style classification pipeline stage — identify CAPTCHA visual style.
|
||||
|
||||
Classifies a CAPTCHA image into one of 6 styles (line-strike, classic, perspective,
|
||||
grid, emboss, color-mesh) using a MobileNetV3-Small classifier. Runs in ~1.5ms
|
||||
Classifies a CAPTCHA image into one of 7 styles (line-strike, classic, perspective,
|
||||
grid, emboss, colorful, color-mesh) using a MobileNetV3-Small classifier. Runs in ~1.5ms
|
||||
on GPU and provides the style context needed for expert model routing.
|
||||
"""
|
||||
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Train the MobileNetV3-Small style classifier for CAPTCHA images.
|
||||
|
||||
Classifies CAPTCHAs into 6 visual styles (line-strike, classic, perspective,
|
||||
grid, emboss, color-mesh) using transfer learning from ImageNet.
|
||||
Classifies CAPTCHAs into 7 visual styles (line-strike, classic, perspective,
|
||||
grid, emboss, colorful, color-mesh) using transfer learning from ImageNet.
|
||||
|
||||
Defaults to disk-based loading from pre-generated dataset (~/.cache/captcha-gen).
|
||||
Falls back to on-the-fly synthetic generation if dataset directory is missing.
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ export function createControlPanelRouter(deps: ControlPanelDeps): Router {
|
|||
registerRateRoutes(router, {
|
||||
pacingEngine,
|
||||
crawlConfig,
|
||||
jobQueue,
|
||||
});
|
||||
|
||||
registerDaemonRoutes(router, {
|
||||
|
|
|
|||
|
|
@ -8,6 +8,8 @@ import { resolve } from 'node:path';
|
|||
import YAML from 'yaml';
|
||||
import { z } from 'zod';
|
||||
|
||||
import { bandwidthTracker } from '../services/bandwidth-tracker';
|
||||
import type { CrawlJobQueue } from '../jobs/crawl-job-queue';
|
||||
import type { PacingEngine } from '../outreach/pacing-engine';
|
||||
import type { CrawlConfig } from '../types';
|
||||
import type { Router, Request, Response } from 'express';
|
||||
|
|
@ -19,6 +21,7 @@ import type { Router, Request, Response } from 'express';
|
|||
interface RateDeps {
|
||||
pacingEngine: PacingEngine;
|
||||
crawlConfig: CrawlConfig;
|
||||
jobQueue: CrawlJobQueue | null;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
|
|
@ -63,7 +66,7 @@ const updateRatesSchema = z.object({
|
|||
// ============================================================================
|
||||
|
||||
export function registerRateRoutes(router: Router, deps: RateDeps): void {
|
||||
const { pacingEngine, crawlConfig } = deps;
|
||||
const { pacingEngine, crawlConfig, jobQueue } = deps;
|
||||
|
||||
/**
|
||||
* GET /api/controlpanel/rates — Get rate configuration (crawl + pacing + safety)
|
||||
|
|
@ -171,4 +174,33 @@ export function registerRateRoutes(router: Router, deps: RateDeps): void {
|
|||
const counts = await pacingEngine.getCounts();
|
||||
res.json({ data: { reset: true, counts } });
|
||||
});
|
||||
|
||||
/**
|
||||
* GET /api/controlpanel/rates/operations-state — Live operations metrics
|
||||
* Returns bandwidth snapshot, request throughput, and crawl concurrency utilization
|
||||
*/
|
||||
router.get('/api/controlpanel/rates/operations-state', async (_req: Request, res: Response) => {
|
||||
const snapshot = bandwidthTracker.getSnapshot();
|
||||
const jobCounts = jobQueue ? await jobQueue.getJobCounts() : null;
|
||||
const activeJobs = jobCounts?.active ?? 0;
|
||||
|
||||
res.json({
|
||||
data: {
|
||||
bandwidth: {
|
||||
currentMbPerHr: snapshot.lastMinuteMbPerHr,
|
||||
peakMbPerHr: snapshot.peakMinuteMbPerHr,
|
||||
requestsLastMinute: snapshot.requestsLastMinute,
|
||||
requestsLast5Min: snapshot.requestsLast5Min,
|
||||
requestsLastHour: snapshot.requestsLastHour,
|
||||
peakRequestsPerMinute: snapshot.peakRequestsPerMinute,
|
||||
utilization: snapshot.utilization,
|
||||
},
|
||||
crawl: {
|
||||
activeJobs,
|
||||
configuredConcurrency: crawlConfig.crawl.concurrency,
|
||||
maxPagesPerCity: crawlConfig.crawl.maxPagesPerCity,
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue