parsers#

import "github.com/magnus-ffcg/go-dbt2lookml/pkg/parsers"

Package parsers provides functionality for parsing dbt manifest and catalog files.

This package handles reading and parsing dbt’s generated JSON files (manifest.json and catalog.json) into Go structs for further processing. It manages the complex task of merging catalog metadata with manifest model definitions.

Key components:

  • DbtParser: Main coordinator for parsing dbt artifacts
  • ModelParser: Parses dbt model definitions
  • CatalogParser: Parses dbt catalog metadata
  • ExposureParser: Parses dbt exposure definitions

The parser supports:

  • BigQuery data types and nested structures
  • Column metadata and descriptions
  • Model relationships and dependencies
  • Custom meta tags for LookML generation
  • Filtering by tags and model selection

Example usage:

cfg := &config.Config{
    ManifestPath: "./target/manifest.json",
    CatalogPath:  "./target/catalog.json",
}
parser := NewDbtParser(cfg)
models, err := parser.Parse()

Index#

type CatalogParser#

CatalogParser handles parsing of DBT catalog data

type CatalogParser struct {
    // contains filtered or unexported fields
}

func NewCatalogParser#

func NewCatalogParser(catalog *models.DbtCatalog, rawCatalog map[string]interface{}, cfg *config.Config) *CatalogParser

NewCatalogParser creates a new CatalogParser instance

func (*CatalogParser) GetCatalogColumn#

func (p *CatalogParser) GetCatalogColumn(modelUniqueID, columnName string) (*models.DbtCatalogNodeColumn, bool)

GetCatalogColumn gets a specific column from the catalog

func (*CatalogParser) GetColumnDataType#

func (p *CatalogParser) GetColumnDataType(modelUniqueID, columnName string) (string, bool)

GetColumnDataType gets the simplified data type for a specific column

func (*CatalogParser) GetColumnInnerTypes#

func (p *CatalogParser) GetColumnInnerTypes(modelUniqueID, columnName string) ([]string, bool)

GetColumnInnerTypes gets the inner types for a specific column (for ARRAY/STRUCT types)

func (*CatalogParser) GetColumnType#

func (p *CatalogParser) GetColumnType(modelUniqueID, columnName string) (string, bool)

GetColumnType gets the BigQuery type for a specific column

func (*CatalogParser) GetModelCatalogData#

func (p *CatalogParser) GetModelCatalogData(modelUniqueID string) (map[string]interface{}, bool)

GetModelCatalogData gets the raw catalog data for a specific model

func (*CatalogParser) GetNestedColumns#

func (p *CatalogParser) GetNestedColumns(modelUniqueID, parentColumnName string) []models.DbtCatalogNodeColumn

GetNestedColumns gets all nested columns for a given parent column

func (*CatalogParser) IsArrayType#

func (p *CatalogParser) IsArrayType(modelUniqueID, columnName string) bool

IsArrayType checks if a column is an ARRAY type

func (*CatalogParser) IsStructType#

func (p *CatalogParser) IsStructType(modelUniqueID, columnName string) bool

IsStructType checks if a column is a STRUCT type

func (*CatalogParser) ProcessModelColumns#

func (p *CatalogParser) ProcessModelColumns(model *models.DbtModel) (*models.DbtModel, error)

ProcessModelColumns processes model columns by merging with catalog information

type DbtParser#

DbtParser is the main DBT parser that coordinates parsing of manifest and catalog files

type DbtParser struct {
    // contains filtered or unexported fields
}

func NewDbtParser#

func NewDbtParser(cliArgs interface{}, rawManifest, rawCatalog map[string]interface{}) (*DbtParser, error)

NewDbtParser creates a new DbtParser instance

func (*DbtParser) GetModels#

func (p *DbtParser) GetModels() ([]*models.DbtModel, error)

GetModels parses dbt models from manifest and filters by criteria

type ExposureParser#

ExposureParser handles parsing of DBT exposures from manifest

type ExposureParser struct {
    // contains filtered or unexported fields
}

func NewExposureParser#

func NewExposureParser(manifest *models.DbtManifest) *ExposureParser

NewExposureParser creates a new ExposureParser instance

func (*ExposureParser) GetAllExposures#

func (p *ExposureParser) GetAllExposures() []models.DbtExposure

GetAllExposures gets all exposures from the manifest

func (*ExposureParser) GetExposureByName#

func (p *ExposureParser) GetExposureByName(name string) (*models.DbtExposure, bool)

GetExposureByName gets a specific exposure by name

func (*ExposureParser) GetExposureModelDependencies#

func (p *ExposureParser) GetExposureModelDependencies(exposureName string) ([]string, bool)

GetExposureModelDependencies gets all model dependencies for an exposure

func (*ExposureParser) GetExposures#

func (p *ExposureParser) GetExposures(tag string) []string

GetExposures gets exposure names, optionally filtered by tag

func (*ExposureParser) GetExposuresByTag#

func (p *ExposureParser) GetExposuresByTag(tag string) []models.DbtExposure

GetExposuresByTag gets all exposures with a specific tag

func (*ExposureParser) ValidateExposureRefs#

func (p *ExposureParser) ValidateExposureRefs(modelNames []string) map[string][]string

ValidateExposureRefs validates that all exposure refs point to existing models

type ModelFilterOptions#

ModelFilterOptions contains options for filtering models

type ModelFilterOptions struct {
    SelectModel   string
    Tag           string
    ExposedNames  []string
    IncludeModels []string
    ExcludeModels []string
}

type ModelParser#

ModelParser handles parsing of DBT models from manifest

type ModelParser struct {
    // contains filtered or unexported fields
}

func NewModelParser#

func NewModelParser(manifest *models.DbtManifest, cfg *config.Config) *ModelParser

NewModelParser creates a new ModelParser instance

func (*ModelParser) FilterModels#

func (p *ModelParser) FilterModels(modelsList []*models.DbtModel, options ModelFilterOptions) []*models.DbtModel

FilterModels filters models based on multiple criteria

func (*ModelParser) GetAllModels#

func (p *ModelParser) GetAllModels() ([]*models.DbtModel, error)

GetAllModels gets all models from manifest

func (*ModelParser) GetModelByName#

func (p *ModelParser) GetModelByName(name string) (*models.DbtModel, error)

GetModelByName gets a specific model by name

func (*ModelParser) GetModelsByTag#

func (p *ModelParser) GetModelsByTag(tag string) ([]*models.DbtModel, error)

GetModelsByTag gets all models with a specific tag

Generated by gomarkdoc

Previous Models Utils Next