stable 2
This commit is contained in:
26
src/index.ts
26
src/index.ts
@@ -2141,16 +2141,24 @@ ${JSON.stringify(schema, null, 2)}`
|
|||||||
const samples = field.samples;
|
const samples = field.samples;
|
||||||
const nonNullSamples = samples.filter(s => s !== null && s !== undefined);
|
const nonNullSamples = samples.filter(s => s !== null && s !== undefined);
|
||||||
|
|
||||||
// Calculate required percentage
|
// Filter out empty/meaningless values for better required field detection
|
||||||
const requiredPercentage = nonNullSamples.length / totalRecords;
|
const meaningfulSamples = samples.filter(s =>
|
||||||
const required = requiredPercentage > 0.8; // 80% threshold for required
|
s !== null &&
|
||||||
|
s !== undefined &&
|
||||||
|
s !== "" &&
|
||||||
|
(typeof s === 'string' ? s.trim() !== "" : true)
|
||||||
|
);
|
||||||
|
|
||||||
|
// Calculate required percentage based on meaningful data
|
||||||
|
const requiredPercentage = meaningfulSamples.length / totalRecords;
|
||||||
|
const required = requiredPercentage > 0.7; // 70% threshold for required (more lenient)
|
||||||
|
|
||||||
// Detect type
|
// Detect type
|
||||||
let type = 'string';
|
let type = 'string';
|
||||||
let size = 255;
|
let size = 255;
|
||||||
|
|
||||||
if (nonNullSamples.length > 0) {
|
if (meaningfulSamples.length > 0) {
|
||||||
const firstSample = nonNullSamples[0];
|
const firstSample = meaningfulSamples[0];
|
||||||
|
|
||||||
if (typeof firstSample === 'boolean') {
|
if (typeof firstSample === 'boolean') {
|
||||||
type = 'boolean';
|
type = 'boolean';
|
||||||
@@ -2158,16 +2166,16 @@ ${JSON.stringify(schema, null, 2)}`
|
|||||||
type = 'integer';
|
type = 'integer';
|
||||||
} else if (typeof firstSample === 'string') {
|
} else if (typeof firstSample === 'string') {
|
||||||
// Check for email pattern
|
// Check for email pattern
|
||||||
if (nonNullSamples.some(s => /^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(s))) {
|
if (meaningfulSamples.some(s => /^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(s))) {
|
||||||
type = 'email';
|
type = 'email';
|
||||||
}
|
}
|
||||||
// Check for datetime pattern
|
// Check for datetime pattern
|
||||||
else if (nonNullSamples.some(s => !isNaN(Date.parse(s)))) {
|
else if (meaningfulSamples.some(s => !isNaN(Date.parse(s)))) {
|
||||||
type = 'datetime';
|
type = 'datetime';
|
||||||
} else {
|
} else {
|
||||||
type = 'string';
|
type = 'string';
|
||||||
// Calculate max string length
|
// Calculate max string length from meaningful samples
|
||||||
const maxLength = Math.max(...nonNullSamples.map(s => String(s).length));
|
const maxLength = Math.max(...meaningfulSamples.map(s => String(s).length));
|
||||||
size = Math.max(255, Math.ceil(maxLength * 1.2)); // 20% buffer
|
size = Math.max(255, Math.ceil(maxLength * 1.2)); // 20% buffer
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user