Skip to content

Skip scanning internal index options from information schema #3767

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jason424217 wants to merge 1 commit into
base: main
Choose a base branch
from
+112 −2
Open

Skip scanning internal index options from information schema #3767

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 10 additions & 2 deletions v1/src/main/java/com/google/cloud/teleport/spanner/ddl/InformationSchemaScanner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -650,7 +650,8 @@ Statement listIndexColumnsSQL() {
}
}

private void listIndexOptions(
@VisibleForTesting
void listIndexOptions(
Comment thread
jason424217 marked this conversation as resolved.
Ddl.Builder builder, Map<String, NavigableMap<String, Index.Builder>> indexes) {
Statement statement = listIndexOptionsSQL();

Expand All @@ -659,9 +660,16 @@ private void listIndexOptions(
Map<KV<String, String>, ImmutableList.Builder<String>> allOptions = Maps.newHashMap();
while (resultSet.next()) {
String tableName = getQualifiedName(resultSet.getString(0), resultSet.getString(1));
String indexName = resultSet.getString(2);
String indexName =
dialect == Dialect.POSTGRESQL
? resultSet.getString(2)
: getQualifiedName(resultSet.getString(0), resultSet.getString(2));
String indexType = resultSet.getString(3);
String optionName = resultSet.getString(4);
if (optionName.startsWith("internal_updated_")
|| optionName.startsWith("system_optimized_")) {
continue;
}
Comment thread
jason424217 marked this conversation as resolved.
String optionType = resultSet.getString(5);
String optionValue = resultSet.getString(6);

Expand Down
102 changes: 102 additions & 0 deletions v1/src/test/java/com/google/cloud/teleport/spanner/ddl/InformationSchemaScannerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,19 @@
package com.google.cloud.teleport.spanner.ddl;

import static org.hamcrest.text.IsEqualCompressingWhiteSpace.equalToCompressingWhiteSpace;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertThrows;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

import com.google.cloud.spanner.Dialect;
import com.google.cloud.spanner.ReadContext;
import com.google.cloud.spanner.ResultSet;
import com.google.common.collect.Maps;
import java.util.Map;
import java.util.NavigableMap;
import org.junit.Test;

/** Unit tests for InformationSchemaScanner class. */
Expand Down Expand Up @@ -111,6 +120,99 @@ public void testListIndexesSQL() {
+ " ORDER BY t.table_name, t.index_name"));
}

@Test
public void testListIndexOptions() {
ReadContext mockContext = mock(ReadContext.class);
ResultSet mockResultSet = mock(ResultSet.class);
when(mockContext.executeQuery(any())).thenReturn(mockResultSet);

when(mockResultSet.next()).thenReturn(true, true, true, false);

// Row 1: Valid option, Row 2: internal_updated_, Row 3: system_optimized_
when(mockResultSet.getString(0)).thenReturn("", "", ""); // schema
when(mockResultSet.getString(1)).thenReturn("table1", "table1", "table1"); // table
when(mockResultSet.getString(2)).thenReturn("index1", "index1", "index1"); // index
when(mockResultSet.getString(3)).thenReturn("INDEX", "INDEX", "INDEX"); // index type
when(mockResultSet.getString(4))
.thenReturn("valid_option", "internal_updated_opt", "system_optimized_opt"); // option name
when(mockResultSet.getString(5)).thenReturn("STRING", "STRING", "STRING"); // option type
when(mockResultSet.getString(6)).thenReturn("value1", "value2", "value3"); // option value

InformationSchemaScanner scanner =
new InformationSchemaScanner(mockContext, Dialect.GOOGLE_STANDARD_SQL);

Map<String, NavigableMap<String, Index.Builder>> indexes = Maps.newHashMap();
NavigableMap<String, Index.Builder> tableIndexes = Maps.newTreeMap();
Index.Builder indexBuilder =
Index.builder(Dialect.GOOGLE_STANDARD_SQL).name("index1").table("table1");
tableIndexes.put("index1", indexBuilder);
indexes.put("table1", tableIndexes);

Ddl.Builder builder = Ddl.builder(Dialect.GOOGLE_STANDARD_SQL);
scanner.listIndexOptions(builder, indexes);

Index index = indexBuilder.build();
assertEquals(1, index.options().size());
assertEquals("valid_option=\"value1\"", index.options().get(0));
}
Comment thread
jason424217 marked this conversation as resolved.

@Test
public void testListIndexOptionsWithNamedSchema() {
ReadContext mockContext = mock(ReadContext.class);
ResultSet mockResultSet = mock(ResultSet.class);
when(mockContext.executeQuery(any())).thenReturn(mockResultSet);

when(mockResultSet.next()).thenReturn(true, false);

when(mockResultSet.getString(0)).thenReturn("myschema"); // schema
when(mockResultSet.getString(1)).thenReturn("table1"); // table
when(mockResultSet.getString(2)).thenReturn("index1"); // index
when(mockResultSet.getString(3)).thenReturn("INDEX"); // index type
when(mockResultSet.getString(4)).thenReturn("valid_option"); // option name
when(mockResultSet.getString(5)).thenReturn("STRING"); // option type
when(mockResultSet.getString(6)).thenReturn("value1"); // option value

InformationSchemaScanner scanner =
new InformationSchemaScanner(mockContext, Dialect.GOOGLE_STANDARD_SQL);

Map<String, NavigableMap<String, Index.Builder>> indexes = Maps.newHashMap();
NavigableMap<String, Index.Builder> tableIndexes = Maps.newTreeMap();
Index.Builder indexBuilder =
Index.builder(Dialect.GOOGLE_STANDARD_SQL).name("myschema.index1").table("myschema.table1");
tableIndexes.put("myschema.index1", indexBuilder);
indexes.put("myschema.table1", tableIndexes);

Ddl.Builder builder = Ddl.builder(Dialect.GOOGLE_STANDARD_SQL);
scanner.listIndexOptions(builder, indexes);

Index index = indexBuilder.build();
assertEquals(1, index.options().size());
assertEquals("valid_option=\"value1\"", index.options().get(0));
}

@Test
public void testListIndexOptionsSQL() {
assertThat(
googleSQLInfoScanner.listIndexOptionsSQL().getSql(),
equalToCompressingWhiteSpace(
"SELECT t.table_schema, t.table_name, t.index_name, t.index_type,"
+ " t.option_name, t.option_type, t.option_value"
+ " FROM information_schema.index_options AS t"
+ " WHERE t.table_schema NOT IN"
+ " ('INFORMATION_SCHEMA', 'SPANNER_SYS')"
+ " ORDER BY t.table_name, t.index_name, t.option_name"));

assertThat(
postgresSQLInfoScanner.listIndexOptionsSQL().getSql(),
equalToCompressingWhiteSpace(
"SELECT t.table_schema, t.table_name, t.index_name, t.index_type,"
+ " t.option_name, t.option_type, t.option_value"
+ " FROM information_schema.index_options AS t"
+ " WHERE t.table_schema NOT IN "
+ " ('information_schema', 'spanner_sys', 'pg_catalog')"
+ " ORDER BY t.table_name, t.index_name, t.option_name"));
}

@Test
public void testListIndexColumnsSQL() {
assertThat(
Expand Down