This commit is contained in:
2025-07-25 18:30:37 +03:00
parent 68b6067824
commit d7c1111762

View File

@@ -2141,16 +2141,24 @@ ${JSON.stringify(schema, null, 2)}`
const samples = field.samples; const samples = field.samples;
const nonNullSamples = samples.filter(s => s !== null && s !== undefined); const nonNullSamples = samples.filter(s => s !== null && s !== undefined);
// Calculate required percentage // Filter out empty/meaningless values for better required field detection
const requiredPercentage = nonNullSamples.length / totalRecords; const meaningfulSamples = samples.filter(s =>
const required = requiredPercentage > 0.8; // 80% threshold for required s !== null &&
s !== undefined &&
s !== "" &&
(typeof s === 'string' ? s.trim() !== "" : true)
);
// Calculate required percentage based on meaningful data
const requiredPercentage = meaningfulSamples.length / totalRecords;
const required = requiredPercentage > 0.7; // 70% threshold for required (more lenient)
// Detect type // Detect type
let type = 'string'; let type = 'string';
let size = 255; let size = 255;
if (nonNullSamples.length > 0) { if (meaningfulSamples.length > 0) {
const firstSample = nonNullSamples[0]; const firstSample = meaningfulSamples[0];
if (typeof firstSample === 'boolean') { if (typeof firstSample === 'boolean') {
type = 'boolean'; type = 'boolean';
@@ -2158,16 +2166,16 @@ ${JSON.stringify(schema, null, 2)}`
type = 'integer'; type = 'integer';
} else if (typeof firstSample === 'string') { } else if (typeof firstSample === 'string') {
// Check for email pattern // Check for email pattern
if (nonNullSamples.some(s => /^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(s))) { if (meaningfulSamples.some(s => /^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(s))) {
type = 'email'; type = 'email';
} }
// Check for datetime pattern // Check for datetime pattern
else if (nonNullSamples.some(s => !isNaN(Date.parse(s)))) { else if (meaningfulSamples.some(s => !isNaN(Date.parse(s)))) {
type = 'datetime'; type = 'datetime';
} else { } else {
type = 'string'; type = 'string';
// Calculate max string length // Calculate max string length from meaningful samples
const maxLength = Math.max(...nonNullSamples.map(s => String(s).length)); const maxLength = Math.max(...meaningfulSamples.map(s => String(s).length));
size = Math.max(255, Math.ceil(maxLength * 1.2)); // 20% buffer size = Math.max(255, Math.ceil(maxLength * 1.2)); // 20% buffer
} }
} }