fix: CSV importer - skip null fields + auto set vertices import

Fixed issue #3818

Author: lvca

integration/src/main/java/com/arcadedb/integration/importer/Importer.java

@@ -50,7 +50,14 @@ public Map<String, Object> load() {
 
       startImporting();
 
-      loadFromSource(settings.url, AnalyzedEntity.EntityType.DATABASE, analyzedSchema);
+      // Determine entity type for the main URL: if vertexType or edgeType is explicitly set, route accordingly
+      AnalyzedEntity.EntityType urlEntityType = AnalyzedEntity.EntityType.DATABASE;
+      if (settings.options.containsKey("vertexType") && settings.vertices == null)
+        urlEntityType = AnalyzedEntity.EntityType.VERTEX;
+      else if (settings.options.containsKey("edgeType") && settings.edges == null)
+        urlEntityType = AnalyzedEntity.EntityType.EDGE;
+
+      loadFromSource(settings.url, urlEntityType, analyzedSchema);
       loadFromSource(settings.documents, AnalyzedEntity.EntityType.DOCUMENT, analyzedSchema);
       loadFromSource(settings.vertices, AnalyzedEntity.EntityType.VERTEX, analyzedSchema);
       loadFromSource(settings.edges, AnalyzedEntity.EntityType.EDGE, analyzedSchema);

integration/src/main/java/com/arcadedb/integration/importer/SourceDiscovery.java

@@ -197,12 +197,16 @@ private FormatImporter analyzeSourceContent(final Parser parser, final AnalyzedE
       break;
 
     case VERTEX:
-      knownFileType = settings.verticesFileType != null ? settings.verticesFileType : getFileTypeByExtension(settings.vertices);
+      knownFileType = settings.verticesFileType != null ?
+          settings.verticesFileType :
+          getFileTypeByExtension(settings.vertices != null ? settings.vertices : settings.url);
       knownDelimiter = settings.verticesDelimiter;
       break;
 
     case EDGE:
-      knownFileType = settings.edgesFileType != null ? settings.edgesFileType : getFileTypeByExtension(settings.edgeTypeName);
+      knownFileType = settings.edgesFileType != null ?
+          settings.edgesFileType :
+          getFileTypeByExtension(settings.edges != null ? settings.edges : settings.url);
       knownDelimiter = settings.edgesDelimiter;
       break;
 

integration/src/main/java/com/arcadedb/integration/importer/format/CSVImporterFormat.java

@@ -127,8 +127,11 @@ private void loadDocuments(final SourceSchema sourceSchema, final Parser parser,
 
         final MutableDocument document = database.newDocument(settings.documentTypeName);
 
-        for (final AnalyzedProperty prop : properties)
-          document.set(prop.getName(), row[prop.getIndex()]);
+        for (final AnalyzedProperty prop : properties) {
+          final String value = row[prop.getIndex()];
+          if (value != null && !value.isEmpty())
+            document.set(prop.getName(), value);
+        }
 
         document.save();
         context.createdDocuments.incrementAndGet();
@@ -154,44 +157,34 @@ private void loadDocuments(final SourceSchema sourceSchema, final Parser parser,
   private void loadVertices(final SourceSchema sourceSchema, final Parser parser, final Database database,
       final ImporterContext context, final ImporterSettings settings) throws ImportException {
 
-    if (settings.typeIdProperty == null) {
-      LogManager.instance()
-          .log(this, Level.INFO, "Property id was not defined. Set `-typeIdProperty <name>`. Importing is aborted", null,
-              settings.vertexTypeName, settings.typeIdProperty);
-      throw new IllegalArgumentException("Property id was not defined. Set `-typeIdProperty <name>`. Importing is aborted");
-    }
-
     final AnalyzedEntity entity = sourceSchema.getSchema().getEntity(settings.vertexTypeName);
     if (entity == null) {
       LogManager.instance().log(this, Level.INFO, "Vertex type '%s' not defined", null, settings.vertexTypeName);
       return;
     }
 
-    final AnalyzedProperty id = entity.getProperty(settings.typeIdProperty);
+    int idIndex = -1;
+    if (settings.typeIdProperty != null) {
+      final AnalyzedProperty id = entity.getProperty(settings.typeIdProperty);
 
-    if (id == null) {
-      LogManager.instance()
-          .log(this, Level.INFO, "Property Id '%s.%s' is null. Importing is aborted", null, settings.vertexTypeName,
-              settings.typeIdProperty);
-      throw new IllegalArgumentException(
-          "Property Id '" + settings.vertexTypeName + "." + settings.typeIdProperty + "' is null. Importing is aborted");
-    }
+      if (id == null) {
+        LogManager.instance()
+            .log(this, Level.INFO, "Property Id '%s.%s' is null. Importing is aborted", null, settings.vertexTypeName,
+                settings.typeIdProperty);
+        throw new IllegalArgumentException(
+            "Property Id '" + settings.vertexTypeName + "." + settings.typeIdProperty + "' is null. Importing is aborted");
+      }
 
-    long expectedVertices = settings.expectedVertices;
-    if (expectedVertices <= 0)
-      expectedVertices = (int) (sourceSchema.getSource().totalSize / entity.getAverageRowLength());
-    if (expectedVertices <= 0)
-      expectedVertices = 1000000;
-    else if (expectedVertices > Integer.MAX_VALUE)
-      expectedVertices = Integer.MAX_VALUE;
-
-    // Ensure the typeIdProperty has a unique index for edge resolution
-    if (!database.getSchema().getType(settings.vertexTypeName).existsProperty(settings.typeIdProperty))
-      database.transaction(
-          () -> database.getSchema().getType(settings.vertexTypeName).createProperty(settings.typeIdProperty, com.arcadedb.schema.Type.STRING));
-    if (database.getSchema().getType(settings.vertexTypeName).getIndexesByProperties(settings.typeIdProperty).isEmpty())
-      database.transaction(
-          () -> database.getSchema().getType(settings.vertexTypeName).createTypeIndex(Schema.INDEX_TYPE.LSM_TREE, true, settings.typeIdProperty));
+      idIndex = id.getIndex();
+
+      // Ensure the typeIdProperty has a unique index for edge resolution
+      if (!database.getSchema().getType(settings.vertexTypeName).existsProperty(settings.typeIdProperty))
+        database.transaction(
+            () -> database.getSchema().getType(settings.vertexTypeName).createProperty(settings.typeIdProperty, com.arcadedb.schema.Type.STRING));
+      if (database.getSchema().getType(settings.vertexTypeName).getIndexesByProperties(settings.typeIdProperty).isEmpty())
+        database.transaction(
+            () -> database.getSchema().getType(settings.vertexTypeName).createTypeIndex(Schema.INDEX_TYPE.LSM_TREE, true, settings.typeIdProperty));
+    }
 
     final AbstractParser<?> csvParser = createCSVParser(settings);
 
@@ -209,8 +202,6 @@ else if (expectedVertices > Integer.MAX_VALUE)
         DatabaseFactory.getDefaultCharset())) {
       csvParser.beginParsing(inputFileReader);
 
-      final int idIndex = id.getIndex();
-
       final List<AnalyzedProperty> properties = new ArrayList<>();
       if (!settings.vertexPropertiesInclude.isEmpty() && !settings.vertexPropertiesInclude.equalsIgnoreCase("*")) {
         final String[] includes = settings.vertexPropertiesInclude.split(",");
@@ -231,18 +222,21 @@ else if (expectedVertices > Integer.MAX_VALUE)
         if (skipEntries > 0 && line < skipEntries)
           continue;
 
-        if (idIndex >= row.length) {
+        if (idIndex >= 0 && idIndex >= row.length) {
           LogManager.instance()
               .log(this, Level.INFO, "Property Id is configured on property %d but cannot be found on current record. Skip it",
                   null, idIndex);
           continue;
         }
 
         final MutableVertex v = database.newVertex(settings.vertexTypeName);
-        v.set(settings.typeIdProperty, row[idIndex]);
+        if (idIndex >= 0)
+          v.set(settings.typeIdProperty, row[idIndex]);
         for (int p = 0; p < properties.size(); ++p) {
           final AnalyzedProperty prop = properties.get(p);
-          v.set(prop.getName(), row[prop.getIndex()]);
+          final String value = row[prop.getIndex()];
+          if (value != null && !value.isEmpty())
+            v.set(prop.getName(), value);
         }
         database.async().createRecord(v, doc -> context.createdVertices.incrementAndGet());
       }

integration/src/test/java/com/arcadedb/integration/importer/CSVImporterIT.java

@@ -20,12 +20,14 @@
 
 import com.arcadedb.database.Database;
 import com.arcadedb.database.DatabaseFactory;
+import com.arcadedb.database.Document;
 import com.arcadedb.graph.Vertex;
 import com.arcadedb.integration.TestHelper;
 import org.junit.jupiter.api.Test;
 
 import java.io.File;
 import java.nio.file.Files;
+import java.util.Set;
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
@@ -345,6 +347,88 @@ void regressionIssue2267SeparateVertexTypeImports() throws Exception {
     TestHelper.checkActiveDatabases();
   }
 
+  /**
+   * Test that sparse CSV import skips null/empty values instead of storing them.
+   * With 4K columns and most being empty, records should only contain properties with actual values.
+   */
+  @Test
+  void importSparseCSVSkipsNullValues() {
+    final String databasePath = "target/databases/test-import-sparse";
+
+    final DatabaseFactory databaseFactory = new DatabaseFactory(databasePath);
+    if (databaseFactory.exists())
+      databaseFactory.open().drop();
+
+    final Database db = databaseFactory.create();
+    try {
+      db.command("sql", """
+          IMPORT DATABASE file://src/test/resources/importer-sparse.csv
+          WITH maxProperties=1000, maxPropertySize=8192
+          """);
+
+      assertThat(db.countType("Document", true)).isEqualTo(3);
+
+      // Check first row: Id=1, Name=Tesla Model 3, Engine=Electric, Seats=5, GPS=true
+      // Color, Doors, Sunroof should NOT be stored
+      db.iterateType("Document", true).forEachRemaining(record -> {
+        final Document doc = record.asDocument();
+        if ("1".equals(doc.getString("Id"))) {
+          assertThat(doc.has("Name")).isTrue();
+          assertThat(doc.has("Engine")).isTrue();
+          assertThat(doc.has("GPS")).isTrue();
+          // These should not be present - they were empty in the CSV
+          assertThat(doc.has("Color")).as("Empty Color should not be stored").isFalse();
+          assertThat(doc.has("Doors")).as("Empty Doors should not be stored").isFalse();
+          assertThat(doc.has("Sunroof")).as("Empty Sunroof should not be stored").isFalse();
+        }
+      });
+    } finally {
+      db.drop();
+    }
+    TestHelper.checkActiveDatabases();
+  }
+
+  /**
+   * Test that IMPORT DATABASE with vertexType creates vertices, not documents.
+   */
+  @Test
+  void importCSVAsVerticesViaVertexType() {
+    final String databasePath = "target/databases/test-import-vertex-type";
+
+    final DatabaseFactory databaseFactory = new DatabaseFactory(databasePath);
+    if (databaseFactory.exists())
+      databaseFactory.open().drop();
+
+    final Database db = databaseFactory.create();
+    try {
+      db.command("sql", """
+          IMPORT DATABASE file://src/test/resources/importer-sparse.csv
+          WITH vertexType=Car, maxProperties=1000, maxPropertySize=8192
+          """);
+
+      // Should create vertices, not documents
+      assertThat(db.getSchema().existsType("Car")).isTrue();
+      assertThat(db.countType("Car", true)).isEqualTo(3);
+
+      // Verify they are actual vertices
+      db.iterateType("Car", true).forEachRemaining(record -> {
+        assertThat(record.asDocument()).isInstanceOf(Vertex.class);
+      });
+
+      // Also verify sparse values are skipped
+      db.iterateType("Car", true).forEachRemaining(record -> {
+        final Vertex v = record.asVertex();
+        if ("1".equals(v.getString("Id"))) {
+          assertThat(v.has("Name")).isTrue();
+          assertThat(v.has("Color")).as("Empty Color should not be stored").isFalse();
+        }
+      });
+    } finally {
+      db.drop();
+    }
+    TestHelper.checkActiveDatabases();
+  }
+
   /**
    * Regression test for GitHub issue #3713: IMPORT DATABASE rejects edgeBidirectional = false
    * When edgeBidirectional=false is specified, the edge type should be created as non-bidirectional

integration/src/test/resources/importer-sparse.csv

@@ -0,0 +1,4 @@
+Id,Name,Color,Engine,Seats,Doors,Sunroof,GPS
+1,Tesla Model 3,,Electric,5,,,true
+2,Ford Mustang,Red,V8,,2,,
+3,Toyota Prius,,Hybrid,5,4,,