Skip to content

Commit

Permalink
tp: final fixes in preparation for mass table migration
Browse files Browse the repository at this point in the history 
Add all the remaining classes/functions which are necessary for a mass
table migration: mainly this is stuff like missing Schema functions,
Iterators etc. Also address some minor remaining issues.

Change-Id: I74b62d27bbe0ec3caaca8598df8c5dabe0537249
  • Loading branch information
@LalitMaganti
LalitMaganti committed Mar 24, 2023
1 parent 746fc2b commit e1d4d44
Show file tree
Hide file tree
Showing 5 changed files with 190 additions and 51 deletions.
7 changes: 4 additions & 3 deletions python/generators/trace_processor_table/public.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 40,8 @@ class ColumnFlag(enum_Flag):
"""
NONE = 0
SORTED = auto()
HIDDEN = auto()
DENSE = auto()
SET_ID = auto()


Expand Down Expand Up @@ -122,17 124,16 @@ class Table:
columns: The columns in this table.
tabledoc: Documentation for this table. Can include documentation overrides
for auto-added columns (i.e. id and type) and aliases added in
|wrapping_sql_view|. parent: The parent ("super-class") table for this
table.
|wrapping_sql_view|.
parent: The parent table for this table. All columns are inherited from the
specified table.
wrapping_sql_view: See |WrappingSqlView|.
"""
class_name: str
sql_name: str
columns: List[Column]
tabledoc: TableDoc
parent: Optional['Table'] = None
tabledoc: Optional[TableDoc] = None
wrapping_sql_view: Optional[WrappingSqlView] = None


Expand Down
158 changes: 150 additions & 8 deletions python/generators/trace_processor_table/serialize.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 19,6 @@
from python.generators.trace_processor_table.public import ColumnFlag
from python.generators.trace_processor_table.util import ParsedTable
from python.generators.trace_processor_table.util import ParsedColumn
from python.generators.trace_processor_table.util import to_cpp_flags


class ColumnSerializer:
Expand Down Expand Up @@ -104,8 103,8 @@ def storage_init(self) -> Optional[str]:
return None

storage = f'ColumnStorage<ColumnType::{self.name}::stored_type>'
# TODO(lalitm): add support for dense columns.
return f'''{self.name}_({storage}::Create<false>())'''
dense = str(ColumnFlag.DENSE in self.flags).lower()
return f'''{self.name}_({storage}::Create<{dense}>())'''

def column_init(self) -> Optional[str]:
if self.is_implicit_id or self.is_implicit_type:
Expand Down Expand Up @@ -158,6 157,41 @@ def storage(self) -> Optional[str]:
name = self.name
return f' ColumnStorage<ColumnType::{name}::stored_type> {name}_;'

def iterator_getter(self) -> Optional[str]:
name = self.name
return f'''
ColumnType::{self.name}::type {name}() const {{
const auto& col = table_->{name}();
return col.GetAtIdx(its_[col.overlay_index()].index());
}}
'''

def iterator_setter(self) -> Optional[str]:
if self.is_implicit_id or self.is_implicit_type:
return None
return f'''
void set_{self.name}(ColumnType::{self.name}::non_optional_type v) {{
auto* col = mutable_table_->mutable_{self.name}();
col->SetAtIdx(its_[col->overlay_index()].index(), v);
}}
'''

def static_schema(self) -> Optional[str]:
if self.is_implicit_id or self.is_implicit_type:
return None
return f'''
schema.columns.emplace_back(Table::Schema::Column{{
"{self.name}", ColumnType::{self.name}::SqlValueType(), false,
{str(ColumnFlag.SORTED in self.flags).lower()},
{str(ColumnFlag.HIDDEN in self.flags).lower()},
{str(ColumnFlag.SET_ID in self.flags).lower()}}});
'''

def row_eq(self) -> Optional[str]:
if self.is_implicit_id or self.is_implicit_type:
return None
return f'ColumnType::{self.name}::Equals({self.name}, other.{self.name})'


class TableSerializer(object):
"""Functions for seralizing a single Table into C ."""
Expand Down Expand Up @@ -209,6 243,7 @@ def row_struct(self) -> str:
ColumnSerializer.parent_row_initializer, delimiter=', ')
row_init = self.foreach_col(
ColumnSerializer.row_initializer, delimiter=',\n ')
row_eq = self.foreach_col(ColumnSerializer.row_eq, delimiter=' &&\n ')
return f'''
struct Row : public {self.parent_class_name}::Row {{
Row({param},
Expand All @@ -218,6 253,10 @@ def row_struct(self) -> str:
type_ = "{self.table.sql_name}";
}}
{self.foreach_col(ColumnSerializer.row_field)}
bool operator==(const {self.table_name}::Row& other) const {{
return type() == other.type() && {row_eq};
}}
}};
'''

Expand Down Expand Up @@ -295,6 334,55 @@ def insert_common(self) -> str:
type_.Append(string_pool_->InternString(row.type()));
'''

def const_iterator(self) -> str:
iterator_getters = self.foreach_col(
ColumnSerializer.iterator_getter, delimiter='\n')
return f'''
class ConstIterator;
class ConstIterator : public macros_internal::AbstractConstIterator<
ConstIterator, {self.table_name}, RowNumber, ConstRowReference> {{
public:
{iterator_getters}
protected:
explicit ConstIterator(const {self.table_name}* table,
std::vector<ColumnStorageOverlay> overlays)
: AbstractConstIterator(table, std::move(overlays)) {{}}
uint32_t CurrentRowNumber() const {{
return its_.back().index();
}}
private:
friend class {self.table_name};
friend class AbstractConstIterator;
}};
'''

def iterator(self) -> str:
iterator_setters = self.foreach_col(
ColumnSerializer.iterator_setter, delimiter='\n')
return f'''
class Iterator : public ConstIterator {{
public:
{iterator_setters}
RowReference row_reference() const {{
return RowReference(mutable_table_, CurrentRowNumber());
}}
private:
friend class {self.table_name};
explicit Iterator({self.table_name}* table,
std::vector<ColumnStorageOverlay> overlays)
: ConstIterator(table, std::move(overlays)),
mutable_table_(table) {{}}
{self.table_name}* mutable_table_ = nullptr;
}};
'''

def serialize(self) -> str:
return f'''
class {self.table_name} : public macros_internal::MacroTable {{
Expand All @@ -307,10 395,6 @@ class {self.table_name} : public macros_internal::MacroTable {{
{self.foreach_col(ColumnSerializer.coltype_enum)}
}};
{self.row_struct().strip()}
struct IdAndRow {{
uint32_t row;
Id id;
}};
struct ColumnFlag {{
{self.foreach_col(ColumnSerializer.flag)}
}};
Expand All @@ -331,11 415,49 @@ class RowNumber : public macros_internal::AbstractRowNumber<
{self.const_row_reference_struct().strip()}
{self.row_reference_struct().strip()}
{self.const_iterator().strip()}
{self.iterator().strip()}
struct IdAndRow {{
Id id;
uint32_t row;
RowReference row_reference;
RowNumber row_number;
}};
{self.constructor().strip()}
~{self.table_name}() override;
static const char* Name() {{ return "{self.table.sql_name}"; }}
static Table::Schema ComputeStaticSchema() {{
Table::Schema schema;
schema.columns.emplace_back(Table::Schema::Column{{
"id", SqlValue::Type::kLong, true, true, false, false}});
schema.columns.emplace_back(Table::Schema::Column{{
"type", SqlValue::Type::kString, false, false, false, false}});
{self.foreach_col(ColumnSerializer.static_schema)}
return schema;
}}
ConstIterator IterateRows() const {{
return ConstIterator(this, CopyOverlays());
}}
Iterator IterateRows() {{ return Iterator(this, CopyOverlays()); }}
ConstIterator FilterToIterator(
const std::vector<Constraint>& cs,
RowMap::OptimizeFor opt = RowMap::OptimizeFor::kMemory) const {{
return ConstIterator(this, FilterAndApplyToOverlays(cs, opt));
}}
Iterator FilterToIterator(
const std::vector<Constraint>& cs,
RowMap::OptimizeFor opt = RowMap::OptimizeFor::kMemory) {{
return Iterator(this, FilterAndApplyToOverlays(cs, opt));
}}
void ShrinkToFit() {{
{self.foreach_col(ColumnSerializer.shrink_to_fit)}
}}
Expand All @@ -356,7 478,8 @@ class RowNumber : public macros_internal::AbstractRowNumber<
{self.insert_common().strip()}
{self.foreach_col(ColumnSerializer.append)}
UpdateSelfOverlayAfterInsert();
return IdAndRow{{row_number, std::move(id)}};
return IdAndRow{{std::move(id), row_number, RowReference(this, row_number),
RowNumber(row_number)}};
}}
{self.foreach_col(ColumnSerializer.accessor)}
Expand Down Expand Up @@ -395,3 518,22 @@ def serialize_header(ifdef_guard: str, tables: List[ParsedTable],
#endif // {ifdef_guard}
'''.strip()


def to_cpp_flags(raw_flag: ColumnFlag) -> str:
"""Converts a ColumnFlag to the C flags which it represents
It is not valid to call this function with ColumnFlag.NONE as in this case
defaults for that column should be implicitly used."""

assert raw_flag != ColumnFlag.NONE
flags = []
if ColumnFlag.SORTED in raw_flag:
flags.append('Column::Flag::kSorted')
if ColumnFlag.HIDDEN in raw_flag:
flags.append('Column::Flag::kHidden')
if ColumnFlag.DENSE in raw_flag:
flags.append('Column::Flag::kDense')
if ColumnFlag.SET_ID in raw_flag:
flags.append('Column::Flag::kSetId')
return ' | '.join(flags)
65 changes: 29 additions & 36 deletions python/generators/trace_processor_table/util.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 19,14 @@
from typing import List
from typing import Set
from typing import Optional
from typing import Union

from python.generators.trace_processor_table.public import Alias
from python.generators.trace_processor_table.public import Column
from python.generators.trace_processor_table.public import ColumnDoc
from python.generators.trace_processor_table.public import ColumnFlag
from python.generators.trace_processor_table.public import CppColumnType
from python.generators.trace_processor_table.public import CppDouble
from python.generators.trace_processor_table.public import CppInt32
from python.generators.trace_processor_table.public import CppInt64
from python.generators.trace_processor_table.public import CppOptional
Expand Down Expand Up @@ -101,6 103,8 @@ def parse_type(self, col_type: CppColumnType) -> ParsedType:
return ParsedType('int32_t')
if isinstance(col_type, CppUint32):
return ParsedType('uint32_t')
if isinstance(col_type, CppDouble):
return ParsedType('double')
if isinstance(col_type, CppString):
return ParsedType('StringPool::Id')

Expand Down Expand Up @@ -161,36 165,24 @@ def public_sql_name(table: Table) -> str:
wrapping_view = table.wrapping_sql_view
return wrapping_view.view_name if wrapping_view else table.sql_name


def to_cpp_flags(raw_flag: ColumnFlag) -> str:
"""Converts a ColumnFlag to the C flags which it represents
It is not valid to call this function with ColumnFlag.NONE as in this case
defaults for that column should be implicitly used."""

assert raw_flag != ColumnFlag.NONE
flags = []
if ColumnFlag.SORTED in raw_flag:
flags.append('Column::Flag::kSorted')
if ColumnFlag.SET_ID in raw_flag:
flags.append('Column::Flag::kSetId')
return ' | '.join(flags)


def _create_implicit_columns_for_root(parsed: ParsedTable
) -> List[ParsedColumn]:
"""Given a root table, returns the implicit id and type columns."""
assert parsed.table.parent is None
table = parsed.table
assert table.parent is None

sql_name = public_sql_name(parsed.table)
sql_name = public_sql_name(table)
id_doc = table.tabledoc.columns.get('id') if table.tabledoc else None
type_doc = table.tabledoc.columns.get('type') if table.tabledoc else None
return [
ParsedColumn(
Column('id', CppSelfTableId(), ColumnFlag.SORTED),
ColumnDoc(doc=f'Unique idenitifier for this {sql_name}.'),
_to_column_doc(id_doc) if id_doc else ColumnDoc(
doc=f'Unique idenitifier for this {sql_name}.'),
is_implicit_id=True),
ParsedColumn(
Column('type', CppString(), ColumnFlag.NONE),
ColumnDoc(doc='''
_to_column_doc(type_doc) if type_doc else ColumnDoc(doc='''
The name of the "most-specific" child table containing this
row.
'''),
Expand Down Expand Up @@ -226,6 218,14 @@ def dfs(t: ParsedTable):
return result


def _to_column_doc(doc: Union[ColumnDoc, str, None]) -> Optional[ColumnDoc]:
"""Cooerces a user specified ColumnDoc or string into a ColumnDoc."""

if doc is None or isinstance(doc, ColumnDoc):
return doc
return ColumnDoc(doc=doc)


def parse_tables_from_files(input_paths: List[str]) -> List[ParsedTable]:
"""Creates a list of tables with the associated paths."""

Expand All @@ -244,29 244,22 @@ def parse_tables_from_files(input_paths: List[str]) -> List[ParsedTable]:
# Create the list of parsed columns
for i, parsed in enumerate(sorted_tables):
parsed_columns: List[ParsedColumn]
table = parsed.table

if parsed.table.parent:
parsed_parent = parsed_tables[parsed.table.parent.class_name]
if table.parent:
parsed_parent = parsed_tables[table.parent.class_name]
parsed_columns = [
dataclasses.replace(c, is_ancestor=True)
for c in parsed_parent.columns
]
else:
parsed_columns = _create_implicit_columns_for_root(parsed)

for c in parsed.table.columns:
doc = parsed.table.tabledoc.columns.get(c.name)
columndoc: Optional[ColumnDoc]
if not doc:
columndoc = None
elif isinstance(doc, ColumnDoc):
columndoc = doc
else:
columndoc = ColumnDoc(doc=doc)
parsed_columns.append(ParsedColumn(c, columndoc))

new_table = dataclasses.replace(parsed, columns=parsed_columns)
parsed_tables[parsed.table.class_name] = new_table
sorted_tables[i] = new_table
for c in table.columns:
doc = table.tabledoc.columns.get(c.name) if table.tabledoc else None
parsed_columns.append(ParsedColumn(c, _to_column_doc(doc)))

sorted_tables[i] = dataclasses.replace(parsed, columns=parsed_columns)
parsed_tables[parsed.table.class_name] = sorted_tables[i]

return sorted_tables
Loading

0 comments on commit e1d4d44

Please sign in to comment.