From 96d041669d0b2c4b20d14fe40503b11079e4b345 Mon Sep 17 00:00:00 2001 From: Evan Robertson Date: Thu, 9 Apr 2026 19:10:27 -0400 Subject: [PATCH 1/7] feat(ast): add dataset/data-source AST types and lexer keywords Add ~20 new lexer keywords for dataset/data-source parsing. Add XmlSerializeOptions, DefineDataset, DefineDataSource, DataRelation, ParentIdRelation, DataSourceBuffer, DataSourceKeys types. Restructure Statement::Create with CreateTarget enum. Restructure Statement::DefineParameter with ParameterType enum. Add xml_options field to DefineTempTable and DefineBuffer. --- crates/oxabl_ast/src/statement.rs | 178 +++++++++++++++++- crates/oxabl_lexer/build.rs | 22 +++ crates/oxabl_lexer/src/kind.rs | 45 +++++ crates/oxabl_parser/src/parser/mod.rs | 12 +- crates/oxabl_parser/src/parser/statements.rs | 182 ++++++++++++++----- crates/oxabl_parser/src/parser/tests.rs | 44 ++--- resources/keyword_overrides.toml | 98 ++++++++++ 7 files changed, 503 insertions(+), 78 deletions(-) diff --git a/crates/oxabl_ast/src/statement.rs b/crates/oxabl_ast/src/statement.rs index c945221..04d5305 100644 --- a/crates/oxabl_ast/src/statement.rs +++ b/crates/oxabl_ast/src/statement.rs @@ -104,9 +104,7 @@ pub enum Statement { /// Define input/output params DefineParameter { direction: ParameterDirection, - name: Identifier, - data_type: DataType, - no_undo: bool, + param_type: ParameterType, }, /// RUN statement — executes an internal procedure or external `.p` file. @@ -182,6 +180,8 @@ pub enum Statement { fields: Vec, /// Index definitions. indexes: Vec, + /// XML and serialization options (NAMESPACE-URI, SERIALIZE-NAME, etc.). + xml_options: XmlSerializeOptions, }, /// DEFINE BUFFER statement. @@ -197,6 +197,8 @@ pub enum Statement { preselect: bool, /// Optional label for error messages. label: Option, + /// XML and serialization options (NAMESPACE-URI, SERIALIZE-NAME, etc.). + xml_options: XmlSerializeOptions, }, /// CATCH block within a DO/REPEAT/FOR block. @@ -363,8 +365,8 @@ pub enum Statement { type_name: String, }, - /// CREATE buffer-name [NO-ERROR]. - Create { buffer: Identifier, no_error: bool }, + /// CREATE statement — record creation or dynamic object creation. + Create { target: CreateTarget, no_error: bool }, /// DELETE buffer-name [NO-ERROR]. Delete { buffer: Identifier, no_error: bool }, @@ -415,6 +417,39 @@ pub enum Statement { /// Simplified: captures name and raw span of unparsed content for formatter round-tripping. DefineFrame { name: Identifier, raw_span: Span }, + /// DEFINE DATASET statement. + /// + /// ```abl + /// DEFINE DATASET dsPerson FOR ttPerson, ttAddress + /// DATA-RELATION drPersonAddr FOR ttPerson, ttAddress + /// RELATION-FIELDS (personId, personId). + /// ``` + DefineDataset { + name: Identifier, + access: Option, + is_static: bool, + is_new_shared: bool, // TODO: Retrofit on DefineTempTable, DefineBuffer, VariableDeclaration + is_shared: bool, // TODO: Retrofit on DefineTempTable, DefineBuffer, VariableDeclaration + serializable: bool, + non_serializable: bool, + xml_options: XmlSerializeOptions, + reference_only: bool, + buffers: Vec, + data_relations: Vec, + parent_id_relations: Vec, + }, + + /// DEFINE DATA-SOURCE statement. + /// + /// `DEFINE DATA-SOURCE dsCustomer FOR Customer.` + DefineDataSource { + name: Identifier, + access: Option, + is_static: bool, + query: Option, + source_buffers: Vec, + }, + /// INPUT/OUTPUT/INPUT-OUTPUT stream I/O statement. StreamIo { direction: StreamDirection, @@ -632,3 +667,136 @@ pub struct AssignPair { /// The value to assign. pub value: Expression, } + +// ============================================================================= +// XML / Serialize options (shared across TEMP-TABLE, BUFFER, DATASET) +// ============================================================================= + +/// XML and serialization options shared by TEMP-TABLE, BUFFER, and DATASET definitions. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct XmlSerializeOptions { + pub namespace_uri: Option, + pub namespace_prefix: Option, + pub xml_node_name: Option, + pub xml_node_type: Option, + pub serialize_name: Option, + pub serialize_hidden: bool, +} + +// ============================================================================= +// Dataset types +// ============================================================================= + +/// A DATA-RELATION clause in a DEFINE DATASET statement. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DataRelation { + pub name: Option, + pub parent_buffer: Identifier, + pub child_buffer: Identifier, + /// Pairs of (parent_field, child_field) from RELATION-FIELDS. + pub relation_fields: Vec<(Identifier, Identifier)>, + pub reposition: bool, + pub nested: bool, + pub foreign_key_hidden: bool, + pub not_active: bool, + pub recursive: bool, +} + +/// A PARENT-ID-RELATION clause in a DEFINE DATASET statement. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ParentIdRelation { + pub name: Option, + pub parent_buffer: Identifier, + pub child_buffer: Identifier, + pub id_field: Identifier, + pub parent_fields_before: Vec, + pub parent_fields_after: Vec, +} + +// ============================================================================= +// Data-source types +// ============================================================================= + +/// A source buffer phrase in a DEFINE DATA-SOURCE statement. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DataSourceBuffer { + pub name: Identifier, + pub keys: Option, +} + +/// KEYS clause in a DATA-SOURCE source buffer phrase. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum DataSourceKeys { + /// KEYS (field1, field2, ...) + Fields(Vec), + /// KEYS (ROWID) + Rowid, +} + +// ============================================================================= +// CREATE target types +// ============================================================================= + +/// Target of a CREATE statement. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum CreateTarget { + /// CREATE buffer-name — record creation or any unrecognized CREATE. + Name(Identifier), + /// CREATE DATASET/DATA-SOURCE/TEMP-TABLE handle [IN WIDGET-POOL pool]. + Handle { + kind: CreateTargetKind, + handle: Identifier, + widget_pool: Option, + }, +} + +/// The type keyword in a CREATE ... handle statement. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CreateTargetKind { + Dataset, + DataSource, + TempTable, +} + +// ============================================================================= +// Parameter types +// ============================================================================= + +/// The type/shape of a DEFINE PARAMETER statement. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ParameterType { + /// Standard variable parameter: DEFINE INPUT PARAMETER name AS type [NO-UNDO]. + Variable { + name: Identifier, + data_type: DataType, + no_undo: bool, + }, + /// Handle-based parameter: TABLE/TABLE-HANDLE/DATASET/DATASET-HANDLE + Handle { + kind: HandleParamKind, + name: Identifier, + passing: HandlePassingOptions, + }, + /// Buffer parameter: DEFINE PARAMETER BUFFER buf FOR table. + Buffer { + name: Identifier, + target: Identifier, + }, +} + +/// Discriminant for handle-based parameter types. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HandleParamKind { + Table, + TableHandle, + Dataset, + DatasetHandle, +} + +/// Passing options for handle-based parameters (APPEND, BIND, BY-VALUE). +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct HandlePassingOptions { + pub append: bool, + pub bind: bool, + pub by_value: bool, +} diff --git a/crates/oxabl_lexer/build.rs b/crates/oxabl_lexer/build.rs index 60360c4..67b5227 100644 --- a/crates/oxabl_lexer/build.rs +++ b/crates/oxabl_lexer/build.rs @@ -88,6 +88,7 @@ fn main() { "bell", "between", "big endian", + "bind", "blank", "blob", "break", @@ -95,6 +96,7 @@ fn main() { "buffer-compare", "buffer-copy", "by", + "by-value", "call", "can do", "can find", @@ -229,6 +231,7 @@ fn main() { "focus", "font", "for", + "foreign-key-hidden", "form", "format", "fram", @@ -355,7 +358,10 @@ fn main() { "modulo", "mouse", "mpe", + "namespace-prefix", + "namespace-uri", "ne", + "nested", "new", "next", "next-prompt", @@ -381,7 +387,9 @@ fn main() { "no-val", "no-validate", "no-wait", + "non-serializable", "not", + "not-active", "now", "null", "num-ali", @@ -416,6 +424,10 @@ fn main() { "page-top", "param", "parameter", + "parent-fields-after", + "parent-fields-before", + "parent-id-field", + "parent-id-relation", "password-field", "pause", "pdbname", @@ -469,6 +481,9 @@ fn main() { "recid", "rect", "rectangle", + "recursive", + "reference-only", + "relation-fields", "release", "repeat", "reposition", @@ -510,6 +525,9 @@ fn main() { "seek", "select", "self", + "serializable", + "serialize-hidden", + "serialize-name", "session", "set", "set-attr-call-type", @@ -531,6 +549,7 @@ fn main() { "stream-io", "system-dialog", "table", + "table-handle", "table-number", "temp-table", "term", @@ -581,6 +600,7 @@ fn main() { "when", "where", "while", + "widget-pool", "window", "window-maxim", "window-maximized", @@ -594,6 +614,8 @@ fn main() { "workfile", "write", "xcode", + "xml-node-name", + "xml-node-type", "xref", "xref-xml", "yes", diff --git a/crates/oxabl_lexer/src/kind.rs b/crates/oxabl_lexer/src/kind.rs index e615a1a..d6ab6b4 100644 --- a/crates/oxabl_lexer/src/kind.rs +++ b/crates/oxabl_lexer/src/kind.rs @@ -155,6 +155,7 @@ pub enum Kind { BufferCopy, BufferCompare, Close, + ParentIdRelation, // Functions Accum, @@ -294,6 +295,27 @@ pub enum Kind { Through, Thru, Append, + NamespaceUri, + NamespacePrefix, + XmlNodeName, + XmlNodeType, + SerializeName, + SerializeHidden, + Serializable, + NonSerializable, + ReferenceOnly, + RelationFields, + Nested, + ForeignKeyHidden, + NotActive, + Recursive, + ParentIdField, + ParentFieldsBefore, + ParentFieldsAfter, + WidgetPool, + TableHandle, + Bind, + ByValue, // Phrases Editing, @@ -558,6 +580,7 @@ pub enum Kind { XrefXml, Yes, Preprop, + } /// Match a string to a keyword Kind @@ -640,6 +663,7 @@ pub fn match_keyword(s: &str) -> Option { "attr" => Some(Kind::AttrSpace), "back" => Some(Kind::Background), "bell" => Some(Kind::Bell), + "bind" => Some(Kind::Bind), "blob" => Some(Kind::Blob), "call" => Some(Kind::Call), "case" => Some(Kind::Case), @@ -846,6 +870,7 @@ pub fn match_keyword(s: &str) -> Option { "memptr" => Some(Kind::Memptr), "method" => Some(Kind::Method), "modulo" => Some(Kind::Modulo), + "nested" => Some(Kind::Nested), "no-fil" => Some(Kind::NoFill), "no-map" => Some(Kind::NoMap), "no-mes" => Some(Kind::NoMessage), @@ -1002,6 +1027,7 @@ pub fn match_keyword(s: &str) -> Option { "availabl" => Some(Kind::Available), "backgrou" => Some(Kind::Background), "before h" => Some(Kind::BeforeHide), + "by-value" => Some(Kind::ByValue), "can find" => Some(Kind::CanFind), "can-find" => Some(Kind::CanFind), "case sen" => Some(Kind::CaseSensitive), @@ -1163,6 +1189,7 @@ pub fn match_keyword(s: &str) -> Option { "protected" => Some(Kind::Protected), "proversio" => Some(Kind::Proversion), "rectangle" => Some(Kind::Rectangle), + "recursive" => Some(Kind::Recursive), "screen-io" => Some(Kind::ScreenIo), "setuserid" => Some(Kind::Setuserid), "show-stat" => Some(Kind::ShowStats), @@ -1233,6 +1260,7 @@ pub fn match_keyword(s: &str) -> Option { "no-message" => Some(Kind::NoMessage), "no-prefetc" => Some(Kind::NoPrefetch), "no-validat" => Some(Kind::NoValidate), + "not-active" => Some(Kind::NotActive), "num-aliase" => Some(Kind::NumAliases), "os-command" => Some(Kind::OsCommand), "page-botto" => Some(Kind::PageBottom), @@ -1322,6 +1350,7 @@ pub fn match_keyword(s: &str) -> Option { "thread-safe" => Some(Kind::ThreadSafe), "transaction" => Some(Kind::Transaction), "unformatted" => Some(Kind::Unformatted), + "widget-pool" => Some(Kind::WidgetPool), _ => None, }, 12 => match lower { @@ -1365,6 +1394,8 @@ pub fn match_keyword(s: &str) -> Option { "rcode-inform" => Some(Kind::RcodeInformation), "sax-complete" => Some(Kind::SaxComplete), "screen-lines" => Some(Kind::ScreenLines), + "serializable" => Some(Kind::Serializable), + "table-handle" => Some(Kind::TableHandle), "table-number" => Some(Kind::TableNumber), "window-maxim" => Some(Kind::WindowMaximized), "window-minim" => Some(Kind::WindowMinimized), @@ -1395,6 +1426,7 @@ pub fn match_keyword(s: &str) -> Option { "is-attr-space" => Some(Kind::IsAttrSpace), "little-endian" => Some(Kind::LittleEndian), "message-lines" => Some(Kind::MessageLines), + "namespace-uri" => Some(Kind::NamespaceUri), "no-attr-space" => Some(Kind::NoAttrSpace), "os-create-dir" => Some(Kind::OsCreateDir), "put-key-value" => Some(Kind::PutKeyValue), @@ -1408,6 +1440,8 @@ pub fn match_keyword(s: &str) -> Option { "window-maximi" => Some(Kind::WindowMaximized), "window-minimi" => Some(Kind::WindowMinimized), "window-normal" => Some(Kind::WindowNormal), + "xml-node-name" => Some(Kind::XmlNodeName), + "xml-node-type" => Some(Kind::XmlNodeType), _ => None, }, 14 => match lower { @@ -1436,7 +1470,9 @@ pub fn match_keyword(s: &str) -> Option { "rcode-informat" => Some(Kind::RcodeInformation), "read-available" => Some(Kind::ReadAvailable), "read-exact-num" => Some(Kind::ReadExactNum), + "reference-only" => Some(Kind::ReferenceOnly), "sax-write-idle" => Some(Kind::SaxWriteIdle), + "serialize-name" => Some(Kind::SerializeName), "this-procedure" => Some(Kind::ThisProcedure), "window-maximiz" => Some(Kind::WindowMaximized), "window-minimiz" => Some(Kind::WindowMinimized), @@ -1453,7 +1489,9 @@ pub fn match_keyword(s: &str) -> Option { "host byte order" => Some(Kind::HostByteOrder), "like-sequential" => Some(Kind::LikeSequential), "package-private" => Some(Kind::PackagePrivate), + "parent-id-field" => Some(Kind::ParentIdField), "rcode-informati" => Some(Kind::RcodeInformation), + "relation-fields" => Some(Kind::RelationFields), "sax-write-begin" => Some(Kind::SaxWriteBegin), "sax-write-error" => Some(Kind::SaxWriteError), "security-policy" => Some(Kind::SecurityPolicy), @@ -1472,8 +1510,11 @@ pub fn match_keyword(s: &str) -> Option { "dynamic-function" => Some(Kind::DynamicFunction), "find wrap around" => Some(Kind::FindWrapAround), "get error column" => Some(Kind::GetErrorColumn), + "namespace-prefix" => Some(Kind::NamespacePrefix), + "non-serializable" => Some(Kind::NonSerializable), "rcode-informatio" => Some(Kind::RcodeInformation), "sax-parser-error" => Some(Kind::SaxParserError), + "serialize-hidden" => Some(Kind::SerializeHidden), "window-maximized" => Some(Kind::WindowMaximized), "window-minimized" => Some(Kind::WindowMinimized), _ => None, @@ -1488,8 +1529,10 @@ pub fn match_keyword(s: &str) -> Option { _ => None, }, 18 => match lower { + "foreign-key-hidden" => Some(Kind::ForeignKeyHidden), "function call type" => Some(Kind::FunctionCallType), "get attr call type" => Some(Kind::GetAttrCallType), + "parent-id-relation" => Some(Kind::ParentIdRelation), "reposition-forward" => Some(Kind::RepositionForward), "sax-write-complete" => Some(Kind::SaxWriteComplete), "set-attr-call-type" => Some(Kind::SetAttrCallType), @@ -1497,6 +1540,7 @@ pub fn match_keyword(s: &str) -> Option { }, 19 => match lower { "find case sensitive" => Some(Kind::FindCaseSensitive), + "parent-fields-after" => Some(Kind::ParentFieldsAfter), "procedure-call-type" => Some(Kind::ProcedureCallType), "reposition-backward" => Some(Kind::RepositionBackward), "reposition-to-rowid" => Some(Kind::RepositionToRowid), @@ -1506,6 +1550,7 @@ pub fn match_keyword(s: &str) -> Option { 20 => match lower { "find next occurrence" => Some(Kind::FindNextOccurrence), "find prev occurrence" => Some(Kind::FindPrevOccurrence), + "parent-fields-before" => Some(Kind::ParentFieldsBefore), _ => None, }, _ => None, diff --git a/crates/oxabl_parser/src/parser/mod.rs b/crates/oxabl_parser/src/parser/mod.rs index 3a32f2d..ee5c37e 100644 --- a/crates/oxabl_parser/src/parser/mod.rs +++ b/crates/oxabl_parser/src/parser/mod.rs @@ -12,7 +12,9 @@ pub mod statements; #[cfg(test)] mod tests; -use oxabl_ast::{AccessModifier, DataType, Identifier, ParameterDirection, Span, Statement}; +use oxabl_ast::{ + AccessModifier, DataType, Identifier, ParameterDirection, ParameterType, Span, Statement, +}; use oxabl_lexer::{Kind, Token, is_callable_kind}; /// An error encountered during parsing, with a human-readable message and source [`Span`]. @@ -388,9 +390,11 @@ impl<'a> Parser<'a> { params.push(Statement::DefineParameter { direction, - name, - data_type, - no_undo, + param_type: ParameterType::Variable { + name, + data_type, + no_undo, + }, }); if !self.check(Kind::Comma) { diff --git a/crates/oxabl_parser/src/parser/statements.rs b/crates/oxabl_parser/src/parser/statements.rs index 185bda3..50dd3b7 100644 --- a/crates/oxabl_parser/src/parser/statements.rs +++ b/crates/oxabl_parser/src/parser/statements.rs @@ -6,10 +6,12 @@ //! LEAVE, NEXT, and RETURN statements. use oxabl_ast::{ - AccessModifier, AssignPair, BufferTarget, DisplayItem, Expression, FieldTypeSource, FindType, - Identifier, IndexField, LockType, ParameterDirection, PreprocIf, RunArgument, RunTarget, - SortDirection, Span, Statement, StreamDirection, StreamOperation, TempTableField, - TempTableIndex, UseIndex, WhenBranch, + AccessModifier, AssignPair, BufferTarget, CreateTarget, CreateTargetKind, DataRelation, + DataSourceBuffer, DataSourceKeys, DisplayItem, Expression, FieldTypeSource, FindType, + HandleParamKind, HandlePassingOptions, Identifier, IndexField, LockType, ParameterDirection, + ParameterType, ParentIdRelation, PreprocIf, RunArgument, RunTarget, SortDirection, Span, + Statement, StreamDirection, StreamOperation, TempTableField, TempTableIndex, UseIndex, + WhenBranch, XmlSerializeOptions, }; use oxabl_lexer::Kind; @@ -460,49 +462,111 @@ impl Parser<'_> { // Expect PARAMETER keyword self.expect_kind(Kind::Parameter, "Expected PARAMETER after INPUT/OUTPUT")?; - // Parse parameter name - if !Self::can_be_identifier(self.peek().kind) { - return Err(ParseError { - message: "Expected parameter name".to_string(), - span: Span { - start: self.peek().start as u32, - end: self.peek().end as u32, - }, - }); - } - let name_token = self.advance().clone(); - let name = Identifier { - span: Span { - start: name_token.start as u32, - end: name_token.end as u32, - }, - name: self.source[name_token.start..name_token.end].to_string(), - }; - - // Expect AS - self.expect_kind(Kind::KwAs, "Expected AS after parameter name")?; - - // Parse data type - let data_type = self.parse_data_type()?; - - // Optional NO-UNDO - let no_undo = if self.check(Kind::NoUndo) { - self.advance(); - true - } else { - false + // Dispatch on parameter type + let param_type = match self.peek().kind { + // TABLE FOR tt-name [APPEND] [BIND] [BY-VALUE] + Kind::Table => { + self.advance(); + self.expect_kind(Kind::KwFor, "Expected FOR after TABLE")?; + let name = self.parse_identifier()?; + let passing = self.parse_handle_passing_options(); + ParameterType::Handle { + kind: HandleParamKind::Table, + name, + passing, + } + } + // TABLE-HANDLE handle [APPEND] [BIND] [BY-VALUE] + Kind::TableHandle => { + self.advance(); + let name = self.parse_identifier()?; + let passing = self.parse_handle_passing_options(); + ParameterType::Handle { + kind: HandleParamKind::TableHandle, + name, + passing, + } + } + // DATASET FOR ds-name [APPEND] [BIND] [BY-VALUE] + Kind::Dataset => { + self.advance(); + self.expect_kind(Kind::KwFor, "Expected FOR after DATASET")?; + let name = self.parse_identifier()?; + let passing = self.parse_handle_passing_options(); + ParameterType::Handle { + kind: HandleParamKind::Dataset, + name, + passing, + } + } + // DATASET-HANDLE handle [APPEND] [BIND] [BY-VALUE] + Kind::DatasetHandle => { + self.advance(); + let name = self.parse_identifier()?; + let passing = self.parse_handle_passing_options(); + ParameterType::Handle { + kind: HandleParamKind::DatasetHandle, + name, + passing, + } + } + // BUFFER buf FOR table + Kind::Buffer => { + self.advance(); + let name = self.parse_identifier()?; + self.expect_kind(Kind::KwFor, "Expected FOR after buffer name")?; + let target = self.parse_identifier()?; + ParameterType::Buffer { name, target } + } + // Standard: name AS type [NO-UNDO] + _ => { + let name = self.parse_identifier()?; + self.expect_kind(Kind::KwAs, "Expected AS after parameter name")?; + let data_type = self.parse_data_type()?; + let no_undo = if self.check(Kind::NoUndo) { + self.advance(); + true + } else { + false + }; + ParameterType::Variable { + name, + data_type, + no_undo, + } + } }; self.expect_kind(Kind::Period, "Expected '.' after parameter definition")?; Ok(Statement::DefineParameter { direction, - name, - data_type, - no_undo, + param_type, }) } + fn parse_handle_passing_options(&mut self) -> HandlePassingOptions { + let mut opts = HandlePassingOptions::default(); + loop { + match self.peek().kind { + Kind::Append => { + self.advance(); + opts.append = true; + } + Kind::Bind => { + self.advance(); + opts.bind = true; + } + Kind::ByValue => { + self.advance(); + opts.by_value = true; + } + _ => break, + } + } + opts + } + // Parse DEFINE TEMP-TABLE fn parse_define_temp_table(&mut self) -> ParseResult { self.advance(); // consume TEMP-TABLE @@ -730,6 +794,7 @@ impl Parser<'_> { use_indexes, fields, indexes, + xml_options: XmlSerializeOptions::default(), }) } @@ -785,6 +850,7 @@ impl Parser<'_> { target, preselect, label, + xml_options: XmlSerializeOptions::default(), }) } @@ -2179,12 +2245,46 @@ impl Parser<'_> { // ========================================================================= // CREATE buffer-name [NO-ERROR]. + // CREATE DATASET/DATA-SOURCE/TEMP-TABLE handle [IN WIDGET-POOL pool] [NO-ERROR]. fn parse_create_statement(&mut self) -> ParseResult { self.advance(); // consume CREATE - let buffer = self.parse_identifier()?; + + let target = if let Some(kind) = self.match_create_target_kind() { + self.advance(); // consume the type keyword + let handle = self.parse_identifier()?; + let widget_pool = self.parse_optional_widget_pool()?; + CreateTarget::Handle { + kind, + handle, + widget_pool, + } + } else { + let name = self.parse_identifier()?; + CreateTarget::Name(name) + }; + let no_error = self.parse_no_error(); self.expect_kind(Kind::Period, "Expected '.' after CREATE statement")?; - Ok(Statement::Create { buffer, no_error }) + Ok(Statement::Create { target, no_error }) + } + + fn match_create_target_kind(&self) -> Option { + match self.peek().kind { + Kind::Dataset => Some(CreateTargetKind::Dataset), + Kind::DataSource => Some(CreateTargetKind::DataSource), + Kind::TempTable => Some(CreateTargetKind::TempTable), + _ => None, + } + } + + fn parse_optional_widget_pool(&mut self) -> ParseResult> { + if self.check(Kind::KwIn) { + self.advance(); // consume IN + self.expect_kind(Kind::WidgetPool, "Expected WIDGET-POOL after IN")?; + Ok(Some(self.parse_expression()?)) + } else { + Ok(None) + } } // DELETE buffer-name [NO-ERROR]. @@ -2265,7 +2365,7 @@ impl Parser<'_> { let target = self.parse_identifier()?; // Optional SAVE RESULT IN clause - // SAVE is Kind::Save, RESULT is an identifier, IN is Kind::KwIn + // SAVE is Kind::Save, RESULT is an identifier, IN is Kind::In let result_var = if self.check(Kind::Save) && self.is_identifier_text_at(1, "RESULT") && self.check_at(2, Kind::KwIn) diff --git a/crates/oxabl_parser/src/parser/tests.rs b/crates/oxabl_parser/src/parser/tests.rs index d5ba82c..27afefa 100644 --- a/crates/oxabl_parser/src/parser/tests.rs +++ b/crates/oxabl_parser/src/parser/tests.rs @@ -1,9 +1,9 @@ use super::*; use oxabl_ast::{ - AccessModifier, BooleanLiteral, BufferTarget, DataType, DecimalLiteral, Expression, - FieldTypeSource, FindType, Identifier, IntegerLiteral, Literal, LockType, ParameterDirection, - RunTarget, SortDirection, Span, Statement, StreamDirection, StreamOperation, StringLiteral, - UnknownLiteral, WhenBranch, + AccessModifier, BooleanLiteral, BufferTarget, CreateTarget, DataType, DecimalLiteral, + Expression, FieldTypeSource, FindType, Identifier, IntegerLiteral, Literal, LockType, + ParameterDirection, ParameterType, RunTarget, SortDirection, Span, Statement, StreamDirection, + StreamOperation, StringLiteral, UnknownLiteral, WhenBranch, }; use oxabl_lexer::tokenize; use rust_decimal::Decimal; @@ -2972,9 +2972,7 @@ fn parse_define_input_parameter() { match stmt { Statement::DefineParameter { direction, - name, - data_type, - no_undo, + param_type: ParameterType::Variable { name, data_type, no_undo }, } => { assert_eq!(direction, ParameterDirection::Input); assert_eq!(name.name, "name"); @@ -2994,9 +2992,7 @@ fn parse_define_output_parameter() { match stmt { Statement::DefineParameter { direction, - name, - data_type, - no_undo, + param_type: ParameterType::Variable { name, data_type, no_undo }, } => { assert_eq!(direction, ParameterDirection::Output); assert_eq!(name.name, "result"); @@ -3016,9 +3012,7 @@ fn parse_define_input_output_parameter() { match stmt { Statement::DefineParameter { direction, - name, - data_type, - no_undo, + param_type: ParameterType::Variable { name, data_type, no_undo }, } => { assert_eq!(direction, ParameterDirection::InputOutput); assert_eq!(name.name, "data"); @@ -3038,9 +3032,7 @@ fn parse_define_parameter_with_no_undo() { match stmt { Statement::DefineParameter { direction, - name, - data_type, - no_undo, + param_type: ParameterType::Variable { name, data_type, no_undo }, } => { assert_eq!(direction, ParameterDirection::Input); assert_eq!(name.name, "name"); @@ -3071,9 +3063,7 @@ END PROCEDURE. match &body[0] { Statement::DefineParameter { direction, - name, - data_type, - .. + param_type: ParameterType::Variable { name, data_type, .. }, } => { assert_eq!(*direction, ParameterDirection::Input); assert_eq!(name.name, "name"); @@ -3085,9 +3075,7 @@ END PROCEDURE. match &body[1] { Statement::DefineParameter { direction, - name, - data_type, - .. + param_type: ParameterType::Variable { name, data_type, .. }, } => { assert_eq!(*direction, ParameterDirection::Output); assert_eq!(name.name, "result"); @@ -5329,8 +5317,8 @@ fn parse_create_basic() { let mut parser = Parser::new(&tokens, source); let stmt = parser.parse_statement().expect("Expected a statement"); match stmt { - Statement::Create { buffer, no_error } => { - assert_eq!(buffer.name, "Customer"); + Statement::Create { target: CreateTarget::Name(name), no_error } => { + assert_eq!(name.name, "Customer"); assert!(!no_error); } _ => panic!("Expected Create statement"), @@ -5344,8 +5332,8 @@ fn parse_create_no_error() { let mut parser = Parser::new(&tokens, source); let stmt = parser.parse_statement().expect("Expected a statement"); match stmt { - Statement::Create { buffer, no_error } => { - assert_eq!(buffer.name, "Customer"); + Statement::Create { target: CreateTarget::Name(name), no_error } => { + assert_eq!(name.name, "Customer"); assert!(no_error); } _ => panic!("Expected Create statement"), @@ -5632,8 +5620,8 @@ fn parse_create_case_insensitive() { let mut parser = Parser::new(&tokens, source); let stmt = parser.parse_statement().expect("Expected a statement"); match stmt { - Statement::Create { buffer, .. } => { - assert_eq!(buffer.name, "customer"); + Statement::Create { target: CreateTarget::Name(name), .. } => { + assert_eq!(name.name, "customer"); } _ => panic!("Expected Create statement"), } diff --git a/resources/keyword_overrides.toml b/resources/keyword_overrides.toml index 2fefb23..2bbee40 100644 --- a/resources/keyword_overrides.toml +++ b/resources/keyword_overrides.toml @@ -511,6 +511,104 @@ keyword_type = "Option" name = "CLOSE" keyword_type = "Statement" +# ============================================================================= +# DATASET / DATA-SOURCE KEYWORDS +# Keywords needed for DEFINE DATASET, DEFINE DATA-SOURCE, and CREATE typed +# variants. XML/serialize keywords are shared with TEMP-TABLE and BUFFER. +# ============================================================================= + +# Shared XML/serialize keywords (TEMP-TABLE, BUFFER, DATASET) +[[add]] +name = "NAMESPACE-URI" +keyword_type = "Option" + +[[add]] +name = "NAMESPACE-PREFIX" +keyword_type = "Option" + +[[add]] +name = "XML-NODE-NAME" +keyword_type = "Option" + +[[add]] +name = "XML-NODE-TYPE" +keyword_type = "Option" + +[[add]] +name = "SERIALIZE-NAME" +keyword_type = "Option" + +[[add]] +name = "SERIALIZE-HIDDEN" +keyword_type = "Option" + +# Dataset-specific keywords +[[add]] +name = "SERIALIZABLE" +keyword_type = "Option" + +[[add]] +name = "NON-SERIALIZABLE" +keyword_type = "Option" + +[[add]] +name = "REFERENCE-ONLY" +keyword_type = "Option" + +[[add]] +name = "RELATION-FIELDS" +keyword_type = "Option" + +[[add]] +name = "NESTED" +keyword_type = "Option" + +[[add]] +name = "FOREIGN-KEY-HIDDEN" +keyword_type = "Option" + +[[add]] +name = "NOT-ACTIVE" +keyword_type = "Option" + +[[add]] +name = "RECURSIVE" +keyword_type = "Option" + +[[add]] +name = "PARENT-ID-RELATION" +keyword_type = "Statement" + +[[add]] +name = "PARENT-ID-FIELD" +keyword_type = "Option" + +[[add]] +name = "PARENT-FIELDS-BEFORE" +keyword_type = "Option" + +[[add]] +name = "PARENT-FIELDS-AFTER" +keyword_type = "Option" + +# CREATE ... IN WIDGET-POOL +[[add]] +name = "WIDGET-POOL" +keyword_type = "Option" + +# DEFINE PARAMETER typed variants +[[add]] +name = "TABLE-HANDLE" +keyword_type = "Option" + +[[add]] +name = "BIND" +keyword_type = "Option" + +[[add]] +name = "BY-VALUE" +keyword_type = "Option" + # ============================================================================= # OVERRIDES # Correct misclassifications or add missing info From 356d6587d6961d1fbc7f2fdbc49cfdc6564b3268 Mon Sep 17 00:00:00 2001 From: Evan Robertson Date: Thu, 9 Apr 2026 19:12:40 -0400 Subject: [PATCH 2/7] feat(parser): add dataset, data-source, and XML/serialize parsing Implement parse_define_dataset() with DATA-RELATION, PARENT-ID-RELATION, RELATION-FIELDS, all flags (REPOSITION, NESTED, FOREIGN-KEY-HIDDEN, NOT-ACTIVE, RECURSIVE), REFERENCE-ONLY, and XML/serialize options. Implement parse_define_data_source() with QUERY and KEYS clauses. Extract parse_xml_serialize_options() helper and retrofit on parse_define_temp_table() and parse_define_buffer(). Restructure parse_create_statement() with CreateTarget dispatch for DATASET, DATA-SOURCE, and TEMP-TABLE with IN WIDGET-POOL. Extend parse_define_parameter() with TABLE, TABLE-HANDLE, DATASET, DATASET-HANDLE, and BUFFER parameter types. Add NEW SHARED/SHARED and SERIALIZABLE/NON-SERIALIZABLE modifier parsing in parse_define_statement() dispatch. --- crates/oxabl_parser/src/parser/statements.rs | 420 ++++++++++++++++++- 1 file changed, 412 insertions(+), 8 deletions(-) diff --git a/crates/oxabl_parser/src/parser/statements.rs b/crates/oxabl_parser/src/parser/statements.rs index 50dd3b7..3d2968a 100644 --- a/crates/oxabl_parser/src/parser/statements.rs +++ b/crates/oxabl_parser/src/parser/statements.rs @@ -273,11 +273,26 @@ impl Parser<'_> { return self.parse_define_parameter(); } - // OO-ABL: DEFINE [access] PROPERTY ... + // Parse optional NEW SHARED / SHARED + let is_new_shared = if self.check(Kind::New) && self.check_at(1, Kind::Shared) { + self.advance(); // consume NEW + self.advance(); // consume SHARED + true + } else { + false + }; + let is_shared = if !is_new_shared && self.check(Kind::Shared) { + self.advance(); + true + } else { + false + }; + + // OO-ABL: DEFINE [access] [STATIC] PROPERTY/DATASET/DATA-SOURCE ... // Check for access modifier before PROPERTY/VARIABLE/TEMP-TABLE let access = self.parse_access_modifier(); - // Check for STATIC after access modifier + // Check for STATIC after access modifier (or before it — both orders valid) let is_static = if self.check(Kind::KwStatic) { self.advance(); true @@ -285,11 +300,32 @@ impl Parser<'_> { false }; + // If STATIC came before access modifier, check for access modifier again + let access = if access.is_none() { + self.parse_access_modifier().or(access) + } else { + access + }; + if self.check(Kind::Property) { return self.parse_define_property(access.unwrap_or(AccessModifier::Public), is_static); } - // If we consumed an access modifier or STATIC but it's not PROPERTY, + // Parse SERIALIZABLE / NON-SERIALIZABLE (dataset-specific, before DATASET keyword) + let serializable = self.check(Kind::Serializable) && { self.advance(); true }; + let non_serializable = !serializable && self.check(Kind::NonSerializable) && { self.advance(); true }; + + // DEFINE DATASET + if self.check(Kind::Dataset) { + return self.parse_define_dataset(access, is_static, is_new_shared, is_shared, serializable, non_serializable); + } + + // DEFINE DATA-SOURCE + if self.check(Kind::DataSource) { + return self.parse_define_data_source(access, is_static); + } + + // If we consumed an access modifier or STATIC but it's not PROPERTY/DATASET/DATA-SOURCE, // fall through to normal DEFINE handling (access modifier is ignored // for VARIABLE/TEMP-TABLE for now — tracked in Future) @@ -319,7 +355,7 @@ impl Parser<'_> { } else { return Err(ParseError { message: - "Expected VARIABLE, VAR, TEMP-TABLE, BUFFER, STREAM, or FRAME after DEFINE" + "Expected VARIABLE, VAR, TEMP-TABLE, BUFFER, STREAM, FRAME, DATASET, or DATA-SOURCE after DEFINE" .to_string(), span: Span { start: self.peek().start as u32, @@ -567,6 +603,78 @@ impl Parser<'_> { opts } + /// Parse XML and serialization options shared by TEMP-TABLE, BUFFER, and DATASET. + /// Consumes NAMESPACE-URI, NAMESPACE-PREFIX, XML-NODE-NAME, XML-NODE-TYPE, + /// SERIALIZE-NAME (all take a string literal or identifier value), + /// and SERIALIZE-HIDDEN (flag, no value). + fn parse_xml_serialize_options(&mut self) -> XmlSerializeOptions { + let mut opts = XmlSerializeOptions::default(); + loop { + match self.peek().kind { + Kind::NamespaceUri => { + self.advance(); + if let Ok(id) = self.parse_string_as_identifier() { + opts.namespace_uri = Some(id); + } + } + Kind::NamespacePrefix => { + self.advance(); + if let Ok(id) = self.parse_string_as_identifier() { + opts.namespace_prefix = Some(id); + } + } + Kind::XmlNodeName => { + self.advance(); + if let Ok(id) = self.parse_string_as_identifier() { + opts.xml_node_name = Some(id); + } + } + Kind::XmlNodeType => { + self.advance(); + if let Ok(id) = self.parse_string_as_identifier() { + opts.xml_node_type = Some(id); + } + } + Kind::SerializeName => { + self.advance(); + if let Ok(id) = self.parse_string_as_identifier() { + opts.serialize_name = Some(id); + } + } + Kind::SerializeHidden => { + self.advance(); + opts.serialize_hidden = true; + } + _ => break, + } + } + opts + } + + /// Parse a string literal as an Identifier (for XML/serialize option values). + fn parse_string_as_identifier(&mut self) -> ParseResult { + if self.check(Kind::StringLiteral) { + let token = self.advance().clone(); + let raw = &self.source[token.start..token.end]; + // Strip quotes from string literal + let name = if raw.len() >= 2 { + raw[1..raw.len() - 1].to_string() + } else { + raw.to_string() + }; + Ok(Identifier { + span: Span { + start: token.start as u32, + end: token.end as u32, + }, + name, + }) + } else { + // Try as regular identifier + self.parse_identifier() + } + } + // Parse DEFINE TEMP-TABLE fn parse_define_temp_table(&mut self) -> ParseResult { self.advance(); // consume TEMP-TABLE @@ -581,6 +689,9 @@ impl Parser<'_> { false }; + // Parse XML/serialize options (NAMESPACE-URI, SERIALIZE-NAME, etc.) + let xml_options = self.parse_xml_serialize_options(); + // Optional LIKE / LIKE-SEQUENTIAL clause let mut like_table = None; let mut validate = false; @@ -794,7 +905,7 @@ impl Parser<'_> { use_indexes, fields, indexes, - xml_options: XmlSerializeOptions::default(), + xml_options, }) } @@ -813,6 +924,9 @@ impl Parser<'_> { BufferTarget::Table(self.parse_identifier()?) }; + // Parse XML/serialize options (NAMESPACE-URI, SERIALIZE-NAME, etc.) + let xml_options = self.parse_xml_serialize_options(); + // Parse optional modifiers let mut preselect = false; let mut label = None; @@ -833,9 +947,8 @@ impl Parser<'_> { } } _ => { - // Skip unknown tokens (NAMESPACE-URI, SERIALIZE-NAME, etc.) + // Skip remaining unknown tokens for forward-compatibility self.advance(); - // Skip their string value if present if self.check(Kind::StringLiteral) { self.advance(); } @@ -850,7 +963,298 @@ impl Parser<'_> { target, preselect, label, - xml_options: XmlSerializeOptions::default(), + xml_options, + }) + } + + // Parse DEFINE DATASET statement. + fn parse_define_dataset( + &mut self, + access: Option, + is_static: bool, + is_new_shared: bool, + is_shared: bool, + serializable: bool, + non_serializable: bool, + ) -> ParseResult { + self.advance(); // consume DATASET + + let name = self.parse_identifier()?; + + // Parse XML/serialize options + let xml_options = self.parse_xml_serialize_options(); + + // Optional REFERENCE-ONLY + let reference_only = if self.check(Kind::ReferenceOnly) { + self.advance(); + true + } else { + false + }; + + // Expect FOR followed by comma-separated buffer names + self.expect_kind(Kind::KwFor, "Expected FOR after dataset name")?; + let mut buffers = vec![self.parse_identifier()?]; + while self.check(Kind::Comma) { + self.advance(); + buffers.push(self.parse_identifier()?); + } + + // Parse DATA-RELATION and PARENT-ID-RELATION clauses + let mut data_relations = Vec::new(); + let mut parent_id_relations = Vec::new(); + + while !self.check(Kind::Period) && !self.at_end() { + if self.check(Kind::DataRelation) { + data_relations.push(self.parse_data_relation()?); + } else if self.check(Kind::ParentIdRelation) { + parent_id_relations.push(self.parse_parent_id_relation()?); + } else if can_start_statement(self.peek().kind) { + return Err(ParseError { + message: "Expected '.' to end DEFINE DATASET (found statement keyword)" + .to_string(), + span: Span { + start: self.peek().start as u32, + end: self.peek().end as u32, + }, + }); + } else { + // Skip unknown tokens for forward-compatibility + self.advance(); + } + } + + self.expect_kind(Kind::Period, "Expected '.' after DEFINE DATASET")?; + + Ok(Statement::DefineDataset { + name, + access, + is_static, + is_new_shared, + is_shared, + serializable, + non_serializable, + xml_options, + reference_only, + buffers, + data_relations, + parent_id_relations, + }) + } + + // Parse a DATA-RELATION clause. + fn parse_data_relation(&mut self) -> ParseResult { + self.advance(); // consume DATA-RELATION + + // Optional relation name (if next token is not FOR) + let name = if !self.check(Kind::KwFor) && Self::can_be_identifier(self.peek().kind) { + Some(self.parse_identifier()?) + } else { + None + }; + + // FOR parent, child + self.expect_kind(Kind::KwFor, "Expected FOR in DATA-RELATION")?; + let parent_buffer = self.parse_identifier()?; + self.expect_kind(Kind::Comma, "Expected ',' between parent and child buffers")?; + let child_buffer = self.parse_identifier()?; + + // RELATION-FIELDS (pf1, cf1 [, pfN, cfN]...) + self.expect_kind( + Kind::RelationFields, + "Expected RELATION-FIELDS in DATA-RELATION", + )?; + self.expect_kind(Kind::LeftParen, "Expected '(' after RELATION-FIELDS")?; + + let mut relation_fields = Vec::new(); + loop { + let parent_field = self.parse_identifier()?; + self.expect_kind(Kind::Comma, "Expected ',' between field pair")?; + let child_field = self.parse_identifier()?; + relation_fields.push((parent_field, child_field)); + if !self.check(Kind::Comma) { + break; + } + self.advance(); // consume comma before next pair + } + self.expect_kind(Kind::RightParen, "Expected ')' after RELATION-FIELDS")?; + + // Parse optional flags + let mut reposition = false; + let mut nested = false; + let mut foreign_key_hidden = false; + let mut not_active = false; + let mut recursive = false; + + loop { + match self.peek().kind { + Kind::Reposition => { + self.advance(); + reposition = true; + } + Kind::Nested => { + self.advance(); + nested = true; + // FOREIGN-KEY-HIDDEN can only follow NESTED + if self.check(Kind::ForeignKeyHidden) { + self.advance(); + foreign_key_hidden = true; + } + } + Kind::NotActive => { + self.advance(); + not_active = true; + } + Kind::Recursive => { + self.advance(); + recursive = true; + } + _ => break, + } + } + + Ok(DataRelation { + name, + parent_buffer, + child_buffer, + relation_fields, + reposition, + nested, + foreign_key_hidden, + not_active, + recursive, + }) + } + + // Parse a PARENT-ID-RELATION clause. + fn parse_parent_id_relation(&mut self) -> ParseResult { + self.advance(); // consume PARENT-ID-RELATION + + // Optional relation name + let name = if !self.check(Kind::KwFor) && Self::can_be_identifier(self.peek().kind) { + Some(self.parse_identifier()?) + } else { + None + }; + + // FOR parent, child + self.expect_kind(Kind::KwFor, "Expected FOR in PARENT-ID-RELATION")?; + let parent_buffer = self.parse_identifier()?; + self.expect_kind(Kind::Comma, "Expected ',' between parent and child buffers")?; + let child_buffer = self.parse_identifier()?; + + // PARENT-ID-FIELD id-field + self.expect_kind( + Kind::ParentIdField, + "Expected PARENT-ID-FIELD in PARENT-ID-RELATION", + )?; + let id_field = self.parse_identifier()?; + + // Optional PARENT-FIELDS-BEFORE + let parent_fields_before = if self.check(Kind::ParentFieldsBefore) { + self.advance(); + self.parse_paren_identifier_list()? + } else { + Vec::new() + }; + + // Optional PARENT-FIELDS-AFTER + let parent_fields_after = if self.check(Kind::ParentFieldsAfter) { + self.advance(); + self.parse_paren_identifier_list()? + } else { + Vec::new() + }; + + Ok(ParentIdRelation { + name, + parent_buffer, + child_buffer, + id_field, + parent_fields_before, + parent_fields_after, + }) + } + + /// Parse a parenthesized comma-separated list of identifiers: (id1, id2, ...) + fn parse_paren_identifier_list(&mut self) -> ParseResult> { + self.expect_kind(Kind::LeftParen, "Expected '('")?; + let mut ids = vec![self.parse_identifier()?]; + while self.check(Kind::Comma) { + self.advance(); + ids.push(self.parse_identifier()?); + } + self.expect_kind(Kind::RightParen, "Expected ')'")?; + Ok(ids) + } + + // Parse DEFINE DATA-SOURCE statement. + fn parse_define_data_source( + &mut self, + access: Option, + is_static: bool, + ) -> ParseResult { + self.advance(); // consume DATA-SOURCE + + let name = self.parse_identifier()?; + + // Expect FOR + self.expect_kind(Kind::KwFor, "Expected FOR after data-source name")?; + + // Optional QUERY query-name + let query = if self.check(Kind::Query) { + self.advance(); + Some(self.parse_identifier()?) + } else { + None + }; + + // Parse comma-separated source buffer phrases + let mut source_buffers = Vec::new(); + loop { + let buf_name = self.parse_identifier()?; + + // Optional KEYS clause + let keys = if self.check(Kind::Keys) { + self.advance(); + self.expect_kind(Kind::LeftParen, "Expected '(' after KEYS")?; + + if self.check(Kind::Rowid) { + self.advance(); + self.expect_kind(Kind::RightParen, "Expected ')' after ROWID")?; + Some(DataSourceKeys::Rowid) + } else { + let mut fields = vec![self.parse_identifier()?]; + while self.check(Kind::Comma) { + self.advance(); + fields.push(self.parse_identifier()?); + } + self.expect_kind(Kind::RightParen, "Expected ')' after KEYS fields")?; + Some(DataSourceKeys::Fields(fields)) + } + } else { + None + }; + + source_buffers.push(DataSourceBuffer { + name: buf_name, + keys, + }); + + if !self.check(Kind::Comma) { + break; + } + self.advance(); // consume comma + } + + self.expect_kind(Kind::Period, "Expected '.' after DEFINE DATA-SOURCE")?; + + Ok(Statement::DefineDataSource { + name, + access, + is_static, + query, + source_buffers, }) } From 27495c214c2a67f7a34cb8c4c017ea2b2ceff1f4 Mon Sep 17 00:00:00 2001 From: Evan Robertson Date: Thu, 9 Apr 2026 19:17:07 -0400 Subject: [PATCH 3/7] test(parser): add dataset, data-source, and parameter tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add 28 new tests covering DEFINE DATASET (all clauses), DEFINE DATA-SOURCE (QUERY, KEYS, ROWID), CREATE typed variants (DATASET, DATA-SOURCE, TEMP-TABLE, Name), DEFINE PARAMETER extensions (TABLE, TABLE-HANDLE, DATASET, DATASET-HANDLE, BUFFER, BIND/APPEND), and XML/serialize option retrofit on DEFINE TEMP-TABLE and DEFINE BUFFER. Fix hyphenated keyword matching for DATA-RELATION, DATA-SOURCE, and DATASET-HANDLE — the HTML keyword index uses space-separated forms but ABL source uses hyphens. --- crates/oxabl_lexer/build.rs | 3 + crates/oxabl_lexer/src/kind.rs | 2 + crates/oxabl_parser/src/parser/tests.rs | 555 +++++++++++++++++++++++- resources/keyword_overrides.toml | 24 + 4 files changed, 581 insertions(+), 3 deletions(-) diff --git a/crates/oxabl_lexer/build.rs b/crates/oxabl_lexer/build.rs index 67b5227..76af0e0 100644 --- a/crates/oxabl_lexer/build.rs +++ b/crates/oxabl_lexer/build.rs @@ -142,10 +142,13 @@ fn main() { "data rel", "data relation", "data source", + "data-relation", + "data-source", "database", "dataservers", "dataset", "dataset handle", + "dataset-handle", "date", "datetime", "datetime-tz", diff --git a/crates/oxabl_lexer/src/kind.rs b/crates/oxabl_lexer/src/kind.rs index d6ab6b4..f09b54b 100644 --- a/crates/oxabl_lexer/src/kind.rs +++ b/crates/oxabl_lexer/src/kind.rs @@ -1416,6 +1416,7 @@ pub fn match_keyword(s: &str) -> Option { "current langu" => Some(Kind::CurrentLanguage), "current-langu" => Some(Kind::CurrentLanguage), "data relation" => Some(Kind::DataRelation), + "data-relation" => Some(Kind::DataRelation), "dbrestriction" => Some(Kind::Dbrestrictions), "get codepages" => Some(Kind::GetCodepages), "get error row" => Some(Kind::GetErrorRow), @@ -1457,6 +1458,7 @@ pub fn match_keyword(s: &str) -> Option { "current-langua" => Some(Kind::CurrentLanguage), "current-window" => Some(Kind::CurrentWindow), "dataset handle" => Some(Kind::DatasetHandle), + "dataset-handle" => Some(Kind::DatasetHandle), "dbrestrictions" => Some(Kind::Dbrestrictions), "default window" => Some(Kind::DefaultWindow), "default-window" => Some(Kind::DefaultWindow), diff --git a/crates/oxabl_parser/src/parser/tests.rs b/crates/oxabl_parser/src/parser/tests.rs index 27afefa..2e4a69b 100644 --- a/crates/oxabl_parser/src/parser/tests.rs +++ b/crates/oxabl_parser/src/parser/tests.rs @@ -1,9 +1,10 @@ use super::*; use oxabl_ast::{ - AccessModifier, BooleanLiteral, BufferTarget, CreateTarget, DataType, DecimalLiteral, - Expression, FieldTypeSource, FindType, Identifier, IntegerLiteral, Literal, LockType, + AccessModifier, BooleanLiteral, BufferTarget, CreateTarget, CreateTargetKind, DataRelation, + DataSourceBuffer, DataSourceKeys, DataType, DecimalLiteral, Expression, FieldTypeSource, + FindType, HandleParamKind, HandlePassingOptions, Identifier, IntegerLiteral, Literal, LockType, ParameterDirection, ParameterType, RunTarget, SortDirection, Span, Statement, StreamDirection, - StreamOperation, StringLiteral, UnknownLiteral, WhenBranch, + StreamOperation, StringLiteral, UnknownLiteral, WhenBranch, XmlSerializeOptions, }; use oxabl_lexer::tokenize; use rust_decimal::Decimal; @@ -6491,3 +6492,551 @@ fn input_as_expression_not_stream() { _ => {} } } + +// ── DEFINE DATASET tests ──────────────────────────────────────── + +#[test] +fn parse_define_dataset_basic() { + let source = "DEFINE DATASET ds FOR ttA."; + let tokens = tokenize(source); + let mut parser = Parser::new(&tokens, source); + let stmt = parser.parse_statement().expect("Expected a statement"); + match stmt { + Statement::DefineDataset { name, buffers, data_relations, parent_id_relations, .. } => { + assert_eq!(name.name, "ds"); + assert_eq!(buffers.len(), 1); + assert_eq!(buffers[0].name, "ttA"); + assert!(data_relations.is_empty()); + assert!(parent_id_relations.is_empty()); + } + _ => panic!("Expected DefineDataset"), + } +} + +#[test] +fn parse_define_dataset_multiple_buffers() { + let source = "DEFINE DATASET ds FOR ttA, ttB, ttC."; + let tokens = tokenize(source); + let mut parser = Parser::new(&tokens, source); + let stmt = parser.parse_statement().expect("Expected a statement"); + match stmt { + Statement::DefineDataset { buffers, .. } => { + assert_eq!(buffers.len(), 3); + assert_eq!(buffers[0].name, "ttA"); + assert_eq!(buffers[1].name, "ttB"); + assert_eq!(buffers[2].name, "ttC"); + } + _ => panic!("Expected DefineDataset"), + } +} + +#[test] +fn parse_define_dataset_data_relation() { + let source = r#"DEFINE DATASET dsPerson FOR ttPerson, ttAddress + DATA-RELATION drPersonAddr FOR ttPerson, ttAddress + RELATION-FIELDS (personId, personId)."#; + let tokens = tokenize(source); + let mut parser = Parser::new(&tokens, source); + let stmt = parser.parse_statement().expect("Expected a statement"); + match stmt { + Statement::DefineDataset { data_relations, .. } => { + assert_eq!(data_relations.len(), 1); + let rel = &data_relations[0]; + assert_eq!(rel.name.as_ref().unwrap().name, "drPersonAddr"); + assert_eq!(rel.parent_buffer.name, "ttPerson"); + assert_eq!(rel.child_buffer.name, "ttAddress"); + assert_eq!(rel.relation_fields.len(), 1); + assert_eq!(rel.relation_fields[0].0.name, "personId"); + assert_eq!(rel.relation_fields[0].1.name, "personId"); + } + _ => panic!("Expected DefineDataset"), + } +} + +#[test] +fn parse_define_dataset_multiple_relations() { + let source = r#"DEFINE DATASET ds FOR ttOrder, ttLine, ttItem + DATA-RELATION r1 FOR ttOrder, ttLine + RELATION-FIELDS (ordNum, ordNum) + DATA-RELATION r2 FOR ttLine, ttItem + RELATION-FIELDS (itemId, itemId)."#; + let tokens = tokenize(source); + let mut parser = Parser::new(&tokens, source); + let stmt = parser.parse_statement().expect("Expected a statement"); + match stmt { + Statement::DefineDataset { data_relations, .. } => { + assert_eq!(data_relations.len(), 2); + assert_eq!(data_relations[0].name.as_ref().unwrap().name, "r1"); + assert_eq!(data_relations[1].name.as_ref().unwrap().name, "r2"); + } + _ => panic!("Expected DefineDataset"), + } +} + +#[test] +fn parse_define_dataset_relation_flags() { + let source = r#"DEFINE DATASET ds FOR ttA, ttB + DATA-RELATION FOR ttA, ttB + RELATION-FIELDS (id, id) + REPOSITION NESTED FOREIGN-KEY-HIDDEN NOT-ACTIVE RECURSIVE."#; + let tokens = tokenize(source); + let mut parser = Parser::new(&tokens, source); + let stmt = parser.parse_statement().expect("Expected a statement"); + match stmt { + Statement::DefineDataset { data_relations, .. } => { + let rel = &data_relations[0]; + assert!(rel.name.is_none()); // unnamed relation + assert!(rel.reposition); + assert!(rel.nested); + assert!(rel.foreign_key_hidden); + assert!(rel.not_active); + assert!(rel.recursive); + } + _ => panic!("Expected DefineDataset"), + } +} + +#[test] +fn parse_define_dataset_parent_id_relation() { + let source = r#"DEFINE DATASET ds FOR ttParent, ttChild + PARENT-ID-RELATION pidRel FOR ttParent, ttChild + PARENT-ID-FIELD idField + PARENT-FIELDS-BEFORE (field1, field2) + PARENT-FIELDS-AFTER (field3)."#; + let tokens = tokenize(source); + let mut parser = Parser::new(&tokens, source); + let stmt = parser.parse_statement().expect("Expected a statement"); + match stmt { + Statement::DefineDataset { parent_id_relations, .. } => { + assert_eq!(parent_id_relations.len(), 1); + let rel = &parent_id_relations[0]; + assert_eq!(rel.name.as_ref().unwrap().name, "pidRel"); + assert_eq!(rel.parent_buffer.name, "ttParent"); + assert_eq!(rel.child_buffer.name, "ttChild"); + assert_eq!(rel.id_field.name, "idField"); + assert_eq!(rel.parent_fields_before.len(), 2); + assert_eq!(rel.parent_fields_before[0].name, "field1"); + assert_eq!(rel.parent_fields_after.len(), 1); + assert_eq!(rel.parent_fields_after[0].name, "field3"); + } + _ => panic!("Expected DefineDataset"), + } +} + +#[test] +fn parse_define_dataset_mixed_relations() { + let source = r#"DEFINE DATASET ds FOR ttA, ttB, ttC + DATA-RELATION FOR ttA, ttB + RELATION-FIELDS (id, id) + PARENT-ID-RELATION FOR ttA, ttC + PARENT-ID-FIELD recid-field."#; + let tokens = tokenize(source); + let mut parser = Parser::new(&tokens, source); + let stmt = parser.parse_statement().expect("Expected a statement"); + match stmt { + Statement::DefineDataset { data_relations, parent_id_relations, .. } => { + assert_eq!(data_relations.len(), 1); + assert_eq!(parent_id_relations.len(), 1); + } + _ => panic!("Expected DefineDataset"), + } +} + +#[test] +fn parse_define_dataset_xml_serialize_options() { + let source = r#"DEFINE DATASET ds + NAMESPACE-URI "urn:example" + NAMESPACE-PREFIX "ex" + XML-NODE-NAME "myDs" + SERIALIZE-NAME "dataset1" + SERIALIZE-HIDDEN + FOR ttA."#; + let tokens = tokenize(source); + let mut parser = Parser::new(&tokens, source); + let stmt = parser.parse_statement().expect("Expected a statement"); + match stmt { + Statement::DefineDataset { xml_options, .. } => { + assert_eq!(xml_options.namespace_uri.as_ref().unwrap().name, "urn:example"); + assert_eq!(xml_options.namespace_prefix.as_ref().unwrap().name, "ex"); + assert_eq!(xml_options.xml_node_name.as_ref().unwrap().name, "myDs"); + assert_eq!(xml_options.serialize_name.as_ref().unwrap().name, "dataset1"); + assert!(xml_options.serialize_hidden); + } + _ => panic!("Expected DefineDataset"), + } +} + +#[test] +fn parse_define_dataset_reference_only() { + let source = "DEFINE DATASET ds REFERENCE-ONLY FOR ttA."; + let tokens = tokenize(source); + let mut parser = Parser::new(&tokens, source); + let stmt = parser.parse_statement().expect("Expected a statement"); + match stmt { + Statement::DefineDataset { reference_only, .. } => { + assert!(reference_only); + } + _ => panic!("Expected DefineDataset"), + } +} + +#[test] +fn parse_define_dataset_modifiers() { + // Test NEW SHARED + let source = "DEFINE NEW SHARED DATASET ds1 FOR ttA."; + let tokens = tokenize(source); + let mut parser = Parser::new(&tokens, source); + let stmt = parser.parse_statement().expect("Expected a statement"); + match stmt { + Statement::DefineDataset { is_new_shared, is_shared, .. } => { + assert!(is_new_shared); + assert!(!is_shared); + } + _ => panic!("Expected DefineDataset"), + } + + // Test PRIVATE STATIC SERIALIZABLE + let source = "DEFINE PRIVATE STATIC SERIALIZABLE DATASET ds2 FOR ttA."; + let tokens = tokenize(source); + let mut parser = Parser::new(&tokens, source); + let stmt = parser.parse_statement().expect("Expected a statement"); + match stmt { + Statement::DefineDataset { access, is_static, serializable, non_serializable, .. } => { + assert_eq!(access, Some(AccessModifier::Private)); + assert!(is_static); + assert!(serializable); + assert!(!non_serializable); + } + _ => panic!("Expected DefineDataset"), + } +} + +// ── DEFINE DATA-SOURCE tests ──────────────────────────────────── + +#[test] +fn parse_define_data_source_basic() { + let source = "DEFINE DATA-SOURCE dsSrc FOR Customer."; + let tokens = tokenize(source); + let mut parser = Parser::new(&tokens, source); + let stmt = parser.parse_statement().expect("Expected a statement"); + match stmt { + Statement::DefineDataSource { name, source_buffers, query, .. } => { + assert_eq!(name.name, "dsSrc"); + assert!(query.is_none()); + assert_eq!(source_buffers.len(), 1); + assert_eq!(source_buffers[0].name.name, "Customer"); + assert!(source_buffers[0].keys.is_none()); + } + _ => panic!("Expected DefineDataSource"), + } +} + +#[test] +fn parse_define_data_source_with_query() { + let source = "DEFINE DATA-SOURCE dsSrc FOR QUERY qCust Customer."; + let tokens = tokenize(source); + let mut parser = Parser::new(&tokens, source); + let stmt = parser.parse_statement().expect("Expected a statement"); + match stmt { + Statement::DefineDataSource { query, source_buffers, .. } => { + assert_eq!(query.as_ref().unwrap().name, "qCust"); + assert_eq!(source_buffers.len(), 1); + assert_eq!(source_buffers[0].name.name, "Customer"); + } + _ => panic!("Expected DefineDataSource"), + } +} + +#[test] +fn parse_define_data_source_multiple_buffers() { + let source = "DEFINE DATA-SOURCE dsSrc FOR Customer, Order."; + let tokens = tokenize(source); + let mut parser = Parser::new(&tokens, source); + let stmt = parser.parse_statement().expect("Expected a statement"); + match stmt { + Statement::DefineDataSource { source_buffers, .. } => { + assert_eq!(source_buffers.len(), 2); + assert_eq!(source_buffers[0].name.name, "Customer"); + assert_eq!(source_buffers[1].name.name, "Order"); + } + _ => panic!("Expected DefineDataSource"), + } +} + +#[test] +fn parse_define_data_source_with_keys() { + let source = "DEFINE DATA-SOURCE dsSrc FOR Customer KEYS (CustNum, Region)."; + let tokens = tokenize(source); + let mut parser = Parser::new(&tokens, source); + let stmt = parser.parse_statement().expect("Expected a statement"); + match stmt { + Statement::DefineDataSource { source_buffers, .. } => { + match &source_buffers[0].keys { + Some(DataSourceKeys::Fields(fields)) => { + assert_eq!(fields.len(), 2); + assert_eq!(fields[0].name, "CustNum"); + assert_eq!(fields[1].name, "Region"); + } + _ => panic!("Expected Fields keys"), + } + } + _ => panic!("Expected DefineDataSource"), + } +} + +#[test] +fn parse_define_data_source_with_rowid_key() { + let source = "DEFINE DATA-SOURCE dsSrc FOR Customer KEYS (ROWID)."; + let tokens = tokenize(source); + let mut parser = Parser::new(&tokens, source); + let stmt = parser.parse_statement().expect("Expected a statement"); + match stmt { + Statement::DefineDataSource { source_buffers, .. } => { + assert!(matches!(source_buffers[0].keys, Some(DataSourceKeys::Rowid))); + } + _ => panic!("Expected DefineDataSource"), + } +} + +#[test] +fn parse_define_data_source_access_static() { + let source = "DEFINE PRIVATE STATIC DATA-SOURCE dsSrc FOR Customer."; + let tokens = tokenize(source); + let mut parser = Parser::new(&tokens, source); + let stmt = parser.parse_statement().expect("Expected a statement"); + match stmt { + Statement::DefineDataSource { access, is_static, .. } => { + assert_eq!(access, Some(AccessModifier::Private)); + assert!(is_static); + } + _ => panic!("Expected DefineDataSource"), + } +} + +// ── CREATE typed tests ────────────────────────────────────────── + +#[test] +fn parse_create_dataset() { + let source = "CREATE DATASET hDs."; + let tokens = tokenize(source); + let mut parser = Parser::new(&tokens, source); + let stmt = parser.parse_statement().expect("Expected a statement"); + match stmt { + Statement::Create { target: CreateTarget::Handle { kind, handle, widget_pool }, .. } => { + assert_eq!(kind, CreateTargetKind::Dataset); + assert_eq!(handle.name, "hDs"); + assert!(widget_pool.is_none()); + } + _ => panic!("Expected Create Dataset"), + } + + // With WIDGET-POOL + let source = r#"CREATE DATASET hDs IN WIDGET-POOL "myPool"."#; + let tokens = tokenize(source); + let mut parser = Parser::new(&tokens, source); + let stmt = parser.parse_statement().expect("Expected a statement"); + match stmt { + Statement::Create { target: CreateTarget::Handle { kind, widget_pool, .. }, .. } => { + assert_eq!(kind, CreateTargetKind::Dataset); + assert!(widget_pool.is_some()); + } + _ => panic!("Expected Create Dataset with widget pool"), + } +} + +#[test] +fn parse_create_data_source() { + let source = "CREATE DATA-SOURCE hDs."; + let tokens = tokenize(source); + let mut parser = Parser::new(&tokens, source); + let stmt = parser.parse_statement().expect("Expected a statement"); + match stmt { + Statement::Create { target: CreateTarget::Handle { kind, handle, .. }, .. } => { + assert_eq!(kind, CreateTargetKind::DataSource); + assert_eq!(handle.name, "hDs"); + } + _ => panic!("Expected Create DataSource"), + } +} + +#[test] +fn parse_create_temp_table() { + let source = "CREATE TEMP-TABLE hTt."; + let tokens = tokenize(source); + let mut parser = Parser::new(&tokens, source); + let stmt = parser.parse_statement().expect("Expected a statement"); + match stmt { + Statement::Create { target: CreateTarget::Handle { kind, handle, .. }, .. } => { + assert_eq!(kind, CreateTargetKind::TempTable); + assert_eq!(handle.name, "hTt"); + } + _ => panic!("Expected Create TempTable"), + } +} + +#[test] +fn parse_create_name() { + // Existing behavior preserved + let source = "CREATE Customer."; + let tokens = tokenize(source); + let mut parser = Parser::new(&tokens, source); + let stmt = parser.parse_statement().expect("Expected a statement"); + match stmt { + Statement::Create { target: CreateTarget::Name(name), no_error } => { + assert_eq!(name.name, "Customer"); + assert!(!no_error); + } + _ => panic!("Expected Create Name"), + } +} + +// ── DEFINE PARAMETER typed tests ──────────────────────────────── + +#[test] +fn parse_define_parameter_dataset() { + let source = "DEFINE INPUT PARAMETER DATASET FOR dsName."; + let tokens = tokenize(source); + let mut parser = Parser::new(&tokens, source); + let stmt = parser.parse_statement().expect("Expected a statement"); + match stmt { + Statement::DefineParameter { + direction, + param_type: ParameterType::Handle { kind, name, passing }, + } => { + assert_eq!(direction, ParameterDirection::Input); + assert_eq!(kind, HandleParamKind::Dataset); + assert_eq!(name.name, "dsName"); + assert!(!passing.append); + assert!(!passing.bind); + } + _ => panic!("Expected DefineParameter Handle Dataset"), + } +} + +#[test] +fn parse_define_parameter_dataset_handle() { + let source = "DEFINE OUTPUT PARAMETER DATASET-HANDLE hDs."; + let tokens = tokenize(source); + let mut parser = Parser::new(&tokens, source); + let stmt = parser.parse_statement().expect("Expected a statement"); + match stmt { + Statement::DefineParameter { + direction, + param_type: ParameterType::Handle { kind, name, .. }, + } => { + assert_eq!(direction, ParameterDirection::Output); + assert_eq!(kind, HandleParamKind::DatasetHandle); + assert_eq!(name.name, "hDs"); + } + _ => panic!("Expected DefineParameter Handle DatasetHandle"), + } +} + +#[test] +fn parse_define_parameter_table_for() { + let source = "DEFINE INPUT PARAMETER TABLE FOR ttName."; + let tokens = tokenize(source); + let mut parser = Parser::new(&tokens, source); + let stmt = parser.parse_statement().expect("Expected a statement"); + match stmt { + Statement::DefineParameter { + param_type: ParameterType::Handle { kind, name, .. }, + .. + } => { + assert_eq!(kind, HandleParamKind::Table); + assert_eq!(name.name, "ttName"); + } + _ => panic!("Expected DefineParameter Handle Table"), + } +} + +#[test] +fn parse_define_parameter_table_handle() { + let source = "DEFINE OUTPUT PARAMETER TABLE-HANDLE hTt."; + let tokens = tokenize(source); + let mut parser = Parser::new(&tokens, source); + let stmt = parser.parse_statement().expect("Expected a statement"); + match stmt { + Statement::DefineParameter { + param_type: ParameterType::Handle { kind, name, .. }, + .. + } => { + assert_eq!(kind, HandleParamKind::TableHandle); + assert_eq!(name.name, "hTt"); + } + _ => panic!("Expected DefineParameter Handle TableHandle"), + } +} + +#[test] +fn parse_define_parameter_buffer() { + let source = "DEFINE INPUT PARAMETER BUFFER bCust FOR Customer."; + let tokens = tokenize(source); + let mut parser = Parser::new(&tokens, source); + let stmt = parser.parse_statement().expect("Expected a statement"); + match stmt { + Statement::DefineParameter { + param_type: ParameterType::Buffer { name, target }, + .. + } => { + assert_eq!(name.name, "bCust"); + assert_eq!(target.name, "Customer"); + } + _ => panic!("Expected DefineParameter Buffer"), + } +} + +#[test] +fn parse_define_parameter_bind_append() { + let source = "DEFINE INPUT PARAMETER DATASET FOR dsName BIND APPEND."; + let tokens = tokenize(source); + let mut parser = Parser::new(&tokens, source); + let stmt = parser.parse_statement().expect("Expected a statement"); + match stmt { + Statement::DefineParameter { + param_type: ParameterType::Handle { passing, .. }, + .. + } => { + assert!(passing.bind); + assert!(passing.append); + assert!(!passing.by_value); + } + _ => panic!("Expected DefineParameter Handle with passing options"), + } +} + +// ── XML/serialize retrofit tests ──────────────────────────────── + +#[test] +fn parse_define_temp_table_xml_options() { + let source = r#"DEFINE TEMP-TABLE tt NO-UNDO + NAMESPACE-URI "urn:test" + SERIALIZE-NAME "myTable" + FIELD x AS INTEGER."#; + let tokens = tokenize(source); + let mut parser = Parser::new(&tokens, source); + let stmt = parser.parse_statement().expect("Expected a statement"); + match stmt { + Statement::DefineTempTable { xml_options, fields, .. } => { + assert_eq!(xml_options.namespace_uri.as_ref().unwrap().name, "urn:test"); + assert_eq!(xml_options.serialize_name.as_ref().unwrap().name, "myTable"); + assert_eq!(fields.len(), 1); + } + _ => panic!("Expected DefineTempTable"), + } +} + +#[test] +fn parse_define_buffer_xml_options() { + let source = r#"DEFINE BUFFER bCust FOR Customer NAMESPACE-URI "urn:foo" SERIALIZE-NAME "cust"."#; + let tokens = tokenize(source); + let mut parser = Parser::new(&tokens, source); + let stmt = parser.parse_statement().expect("Expected a statement"); + match stmt { + Statement::DefineBuffer { xml_options, .. } => { + assert_eq!(xml_options.namespace_uri.as_ref().unwrap().name, "urn:foo"); + assert_eq!(xml_options.serialize_name.as_ref().unwrap().name, "cust"); + } + _ => panic!("Expected DefineBuffer"), + } +} diff --git a/resources/keyword_overrides.toml b/resources/keyword_overrides.toml index 2bbee40..4d97912 100644 --- a/resources/keyword_overrides.toml +++ b/resources/keyword_overrides.toml @@ -609,6 +609,30 @@ keyword_type = "Option" name = "BY-VALUE" keyword_type = "Option" +# ============================================================================= +# HYPHENATED FORM OVERRIDES +# The HTML keyword index uses space-separated forms (e.g., "data relation") +# but ABL source code uses hyphenated forms (e.g., "data-relation"). +# The lexer tokenizes hyphenated identifiers as single tokens, so the +# match_keyword function needs the hyphenated form. +# ============================================================================= + +# DATA-RELATION exists as "data relation" (space) from the HTML index. +# Override to add the hyphenated form that the lexer actually sees. +[[add]] +name = "DATA-RELATION" +keyword_type = "Option" + +# DATA-SOURCE exists as "data source" (space) from the HTML index. +[[add]] +name = "DATA-SOURCE" +keyword_type = "Statement" + +# DATASET-HANDLE exists as "dataset handle" (space) from the HTML index. +[[add]] +name = "DATASET-HANDLE" +keyword_type = "Type" + # ============================================================================= # OVERRIDES # Correct misclassifications or add missing info From e2510d582b67d7906fde12ba94a91ba341c424c4 Mon Sep 17 00:00:00 2001 From: Evan Robertson Date: Thu, 9 Apr 2026 19:18:12 -0400 Subject: [PATCH 4/7] feat(parser): add dataset benchmark fixture and apply formatting Add bench_parser_datasets.abl with diverse dataset/data-source definitions for CodSpeed regression detection. Register in parser_bench.rs. Apply cargo fmt across all changed files. --- crates/oxabl_ast/src/statement.rs | 5 +- crates/oxabl_lexer/src/kind.rs | 1 - crates/oxabl_parser/benches/parser_bench.rs | 1 + crates/oxabl_parser/src/parser/statements.rs | 19 +- crates/oxabl_parser/src/parser/tests.rs | 182 +++++++++++++++---- resources/bench_parser_datasets.abl | 88 +++++++++ 6 files changed, 254 insertions(+), 42 deletions(-) create mode 100644 resources/bench_parser_datasets.abl diff --git a/crates/oxabl_ast/src/statement.rs b/crates/oxabl_ast/src/statement.rs index 04d5305..18f39d4 100644 --- a/crates/oxabl_ast/src/statement.rs +++ b/crates/oxabl_ast/src/statement.rs @@ -366,7 +366,10 @@ pub enum Statement { }, /// CREATE statement — record creation or dynamic object creation. - Create { target: CreateTarget, no_error: bool }, + Create { + target: CreateTarget, + no_error: bool, + }, /// DELETE buffer-name [NO-ERROR]. Delete { buffer: Identifier, no_error: bool }, diff --git a/crates/oxabl_lexer/src/kind.rs b/crates/oxabl_lexer/src/kind.rs index f09b54b..7fcd6e2 100644 --- a/crates/oxabl_lexer/src/kind.rs +++ b/crates/oxabl_lexer/src/kind.rs @@ -580,7 +580,6 @@ pub enum Kind { XrefXml, Yes, Preprop, - } /// Match a string to a keyword Kind diff --git a/crates/oxabl_parser/benches/parser_bench.rs b/crates/oxabl_parser/benches/parser_bench.rs index c8a8a97..38fbf55 100644 --- a/crates/oxabl_parser/benches/parser_bench.rs +++ b/crates/oxabl_parser/benches/parser_bench.rs @@ -43,6 +43,7 @@ fn parser_benchmarks(c: &mut Criterion) { bench_fixture(&mut group, "oo_abl", "bench_parser_oo_abl.abl"); bench_fixture(&mut group, "temp_tables", "bench_parser_temp_tables.abl"); bench_fixture(&mut group, "procs_funcs", "bench_parser_procs_funcs.abl"); + bench_fixture(&mut group, "datasets", "bench_parser_datasets.abl"); group.finish(); } diff --git a/crates/oxabl_parser/src/parser/statements.rs b/crates/oxabl_parser/src/parser/statements.rs index 3d2968a..36851bd 100644 --- a/crates/oxabl_parser/src/parser/statements.rs +++ b/crates/oxabl_parser/src/parser/statements.rs @@ -312,12 +312,25 @@ impl Parser<'_> { } // Parse SERIALIZABLE / NON-SERIALIZABLE (dataset-specific, before DATASET keyword) - let serializable = self.check(Kind::Serializable) && { self.advance(); true }; - let non_serializable = !serializable && self.check(Kind::NonSerializable) && { self.advance(); true }; + let serializable = self.check(Kind::Serializable) && { + self.advance(); + true + }; + let non_serializable = !serializable && self.check(Kind::NonSerializable) && { + self.advance(); + true + }; // DEFINE DATASET if self.check(Kind::Dataset) { - return self.parse_define_dataset(access, is_static, is_new_shared, is_shared, serializable, non_serializable); + return self.parse_define_dataset( + access, + is_static, + is_new_shared, + is_shared, + serializable, + non_serializable, + ); } // DEFINE DATA-SOURCE diff --git a/crates/oxabl_parser/src/parser/tests.rs b/crates/oxabl_parser/src/parser/tests.rs index 2e4a69b..9500e93 100644 --- a/crates/oxabl_parser/src/parser/tests.rs +++ b/crates/oxabl_parser/src/parser/tests.rs @@ -2973,7 +2973,12 @@ fn parse_define_input_parameter() { match stmt { Statement::DefineParameter { direction, - param_type: ParameterType::Variable { name, data_type, no_undo }, + param_type: + ParameterType::Variable { + name, + data_type, + no_undo, + }, } => { assert_eq!(direction, ParameterDirection::Input); assert_eq!(name.name, "name"); @@ -2993,7 +2998,12 @@ fn parse_define_output_parameter() { match stmt { Statement::DefineParameter { direction, - param_type: ParameterType::Variable { name, data_type, no_undo }, + param_type: + ParameterType::Variable { + name, + data_type, + no_undo, + }, } => { assert_eq!(direction, ParameterDirection::Output); assert_eq!(name.name, "result"); @@ -3013,7 +3023,12 @@ fn parse_define_input_output_parameter() { match stmt { Statement::DefineParameter { direction, - param_type: ParameterType::Variable { name, data_type, no_undo }, + param_type: + ParameterType::Variable { + name, + data_type, + no_undo, + }, } => { assert_eq!(direction, ParameterDirection::InputOutput); assert_eq!(name.name, "data"); @@ -3033,7 +3048,12 @@ fn parse_define_parameter_with_no_undo() { match stmt { Statement::DefineParameter { direction, - param_type: ParameterType::Variable { name, data_type, no_undo }, + param_type: + ParameterType::Variable { + name, + data_type, + no_undo, + }, } => { assert_eq!(direction, ParameterDirection::Input); assert_eq!(name.name, "name"); @@ -3064,7 +3084,10 @@ END PROCEDURE. match &body[0] { Statement::DefineParameter { direction, - param_type: ParameterType::Variable { name, data_type, .. }, + param_type: + ParameterType::Variable { + name, data_type, .. + }, } => { assert_eq!(*direction, ParameterDirection::Input); assert_eq!(name.name, "name"); @@ -3076,7 +3099,10 @@ END PROCEDURE. match &body[1] { Statement::DefineParameter { direction, - param_type: ParameterType::Variable { name, data_type, .. }, + param_type: + ParameterType::Variable { + name, data_type, .. + }, } => { assert_eq!(*direction, ParameterDirection::Output); assert_eq!(name.name, "result"); @@ -5318,7 +5344,10 @@ fn parse_create_basic() { let mut parser = Parser::new(&tokens, source); let stmt = parser.parse_statement().expect("Expected a statement"); match stmt { - Statement::Create { target: CreateTarget::Name(name), no_error } => { + Statement::Create { + target: CreateTarget::Name(name), + no_error, + } => { assert_eq!(name.name, "Customer"); assert!(!no_error); } @@ -5333,7 +5362,10 @@ fn parse_create_no_error() { let mut parser = Parser::new(&tokens, source); let stmt = parser.parse_statement().expect("Expected a statement"); match stmt { - Statement::Create { target: CreateTarget::Name(name), no_error } => { + Statement::Create { + target: CreateTarget::Name(name), + no_error, + } => { assert_eq!(name.name, "Customer"); assert!(no_error); } @@ -5621,7 +5653,10 @@ fn parse_create_case_insensitive() { let mut parser = Parser::new(&tokens, source); let stmt = parser.parse_statement().expect("Expected a statement"); match stmt { - Statement::Create { target: CreateTarget::Name(name), .. } => { + Statement::Create { + target: CreateTarget::Name(name), + .. + } => { assert_eq!(name.name, "customer"); } _ => panic!("Expected Create statement"), @@ -6502,7 +6537,13 @@ fn parse_define_dataset_basic() { let mut parser = Parser::new(&tokens, source); let stmt = parser.parse_statement().expect("Expected a statement"); match stmt { - Statement::DefineDataset { name, buffers, data_relations, parent_id_relations, .. } => { + Statement::DefineDataset { + name, + buffers, + data_relations, + parent_id_relations, + .. + } => { assert_eq!(name.name, "ds"); assert_eq!(buffers.len(), 1); assert_eq!(buffers[0].name, "ttA"); @@ -6607,7 +6648,10 @@ fn parse_define_dataset_parent_id_relation() { let mut parser = Parser::new(&tokens, source); let stmt = parser.parse_statement().expect("Expected a statement"); match stmt { - Statement::DefineDataset { parent_id_relations, .. } => { + Statement::DefineDataset { + parent_id_relations, + .. + } => { assert_eq!(parent_id_relations.len(), 1); let rel = &parent_id_relations[0]; assert_eq!(rel.name.as_ref().unwrap().name, "pidRel"); @@ -6634,7 +6678,11 @@ fn parse_define_dataset_mixed_relations() { let mut parser = Parser::new(&tokens, source); let stmt = parser.parse_statement().expect("Expected a statement"); match stmt { - Statement::DefineDataset { data_relations, parent_id_relations, .. } => { + Statement::DefineDataset { + data_relations, + parent_id_relations, + .. + } => { assert_eq!(data_relations.len(), 1); assert_eq!(parent_id_relations.len(), 1); } @@ -6656,10 +6704,16 @@ fn parse_define_dataset_xml_serialize_options() { let stmt = parser.parse_statement().expect("Expected a statement"); match stmt { Statement::DefineDataset { xml_options, .. } => { - assert_eq!(xml_options.namespace_uri.as_ref().unwrap().name, "urn:example"); + assert_eq!( + xml_options.namespace_uri.as_ref().unwrap().name, + "urn:example" + ); assert_eq!(xml_options.namespace_prefix.as_ref().unwrap().name, "ex"); assert_eq!(xml_options.xml_node_name.as_ref().unwrap().name, "myDs"); - assert_eq!(xml_options.serialize_name.as_ref().unwrap().name, "dataset1"); + assert_eq!( + xml_options.serialize_name.as_ref().unwrap().name, + "dataset1" + ); assert!(xml_options.serialize_hidden); } _ => panic!("Expected DefineDataset"), @@ -6688,7 +6742,11 @@ fn parse_define_dataset_modifiers() { let mut parser = Parser::new(&tokens, source); let stmt = parser.parse_statement().expect("Expected a statement"); match stmt { - Statement::DefineDataset { is_new_shared, is_shared, .. } => { + Statement::DefineDataset { + is_new_shared, + is_shared, + .. + } => { assert!(is_new_shared); assert!(!is_shared); } @@ -6701,7 +6759,13 @@ fn parse_define_dataset_modifiers() { let mut parser = Parser::new(&tokens, source); let stmt = parser.parse_statement().expect("Expected a statement"); match stmt { - Statement::DefineDataset { access, is_static, serializable, non_serializable, .. } => { + Statement::DefineDataset { + access, + is_static, + serializable, + non_serializable, + .. + } => { assert_eq!(access, Some(AccessModifier::Private)); assert!(is_static); assert!(serializable); @@ -6720,7 +6784,12 @@ fn parse_define_data_source_basic() { let mut parser = Parser::new(&tokens, source); let stmt = parser.parse_statement().expect("Expected a statement"); match stmt { - Statement::DefineDataSource { name, source_buffers, query, .. } => { + Statement::DefineDataSource { + name, + source_buffers, + query, + .. + } => { assert_eq!(name.name, "dsSrc"); assert!(query.is_none()); assert_eq!(source_buffers.len(), 1); @@ -6738,7 +6807,11 @@ fn parse_define_data_source_with_query() { let mut parser = Parser::new(&tokens, source); let stmt = parser.parse_statement().expect("Expected a statement"); match stmt { - Statement::DefineDataSource { query, source_buffers, .. } => { + Statement::DefineDataSource { + query, + source_buffers, + .. + } => { assert_eq!(query.as_ref().unwrap().name, "qCust"); assert_eq!(source_buffers.len(), 1); assert_eq!(source_buffers[0].name.name, "Customer"); @@ -6770,16 +6843,14 @@ fn parse_define_data_source_with_keys() { let mut parser = Parser::new(&tokens, source); let stmt = parser.parse_statement().expect("Expected a statement"); match stmt { - Statement::DefineDataSource { source_buffers, .. } => { - match &source_buffers[0].keys { - Some(DataSourceKeys::Fields(fields)) => { - assert_eq!(fields.len(), 2); - assert_eq!(fields[0].name, "CustNum"); - assert_eq!(fields[1].name, "Region"); - } - _ => panic!("Expected Fields keys"), + Statement::DefineDataSource { source_buffers, .. } => match &source_buffers[0].keys { + Some(DataSourceKeys::Fields(fields)) => { + assert_eq!(fields.len(), 2); + assert_eq!(fields[0].name, "CustNum"); + assert_eq!(fields[1].name, "Region"); } - } + _ => panic!("Expected Fields keys"), + }, _ => panic!("Expected DefineDataSource"), } } @@ -6792,7 +6863,10 @@ fn parse_define_data_source_with_rowid_key() { let stmt = parser.parse_statement().expect("Expected a statement"); match stmt { Statement::DefineDataSource { source_buffers, .. } => { - assert!(matches!(source_buffers[0].keys, Some(DataSourceKeys::Rowid))); + assert!(matches!( + source_buffers[0].keys, + Some(DataSourceKeys::Rowid) + )); } _ => panic!("Expected DefineDataSource"), } @@ -6805,7 +6879,9 @@ fn parse_define_data_source_access_static() { let mut parser = Parser::new(&tokens, source); let stmt = parser.parse_statement().expect("Expected a statement"); match stmt { - Statement::DefineDataSource { access, is_static, .. } => { + Statement::DefineDataSource { + access, is_static, .. + } => { assert_eq!(access, Some(AccessModifier::Private)); assert!(is_static); } @@ -6822,7 +6898,15 @@ fn parse_create_dataset() { let mut parser = Parser::new(&tokens, source); let stmt = parser.parse_statement().expect("Expected a statement"); match stmt { - Statement::Create { target: CreateTarget::Handle { kind, handle, widget_pool }, .. } => { + Statement::Create { + target: + CreateTarget::Handle { + kind, + handle, + widget_pool, + }, + .. + } => { assert_eq!(kind, CreateTargetKind::Dataset); assert_eq!(handle.name, "hDs"); assert!(widget_pool.is_none()); @@ -6836,7 +6920,12 @@ fn parse_create_dataset() { let mut parser = Parser::new(&tokens, source); let stmt = parser.parse_statement().expect("Expected a statement"); match stmt { - Statement::Create { target: CreateTarget::Handle { kind, widget_pool, .. }, .. } => { + Statement::Create { + target: CreateTarget::Handle { + kind, widget_pool, .. + }, + .. + } => { assert_eq!(kind, CreateTargetKind::Dataset); assert!(widget_pool.is_some()); } @@ -6851,7 +6940,10 @@ fn parse_create_data_source() { let mut parser = Parser::new(&tokens, source); let stmt = parser.parse_statement().expect("Expected a statement"); match stmt { - Statement::Create { target: CreateTarget::Handle { kind, handle, .. }, .. } => { + Statement::Create { + target: CreateTarget::Handle { kind, handle, .. }, + .. + } => { assert_eq!(kind, CreateTargetKind::DataSource); assert_eq!(handle.name, "hDs"); } @@ -6866,7 +6958,10 @@ fn parse_create_temp_table() { let mut parser = Parser::new(&tokens, source); let stmt = parser.parse_statement().expect("Expected a statement"); match stmt { - Statement::Create { target: CreateTarget::Handle { kind, handle, .. }, .. } => { + Statement::Create { + target: CreateTarget::Handle { kind, handle, .. }, + .. + } => { assert_eq!(kind, CreateTargetKind::TempTable); assert_eq!(handle.name, "hTt"); } @@ -6882,7 +6977,10 @@ fn parse_create_name() { let mut parser = Parser::new(&tokens, source); let stmt = parser.parse_statement().expect("Expected a statement"); match stmt { - Statement::Create { target: CreateTarget::Name(name), no_error } => { + Statement::Create { + target: CreateTarget::Name(name), + no_error, + } => { assert_eq!(name.name, "Customer"); assert!(!no_error); } @@ -6901,7 +6999,12 @@ fn parse_define_parameter_dataset() { match stmt { Statement::DefineParameter { direction, - param_type: ParameterType::Handle { kind, name, passing }, + param_type: + ParameterType::Handle { + kind, + name, + passing, + }, } => { assert_eq!(direction, ParameterDirection::Input); assert_eq!(kind, HandleParamKind::Dataset); @@ -7017,7 +7120,11 @@ fn parse_define_temp_table_xml_options() { let mut parser = Parser::new(&tokens, source); let stmt = parser.parse_statement().expect("Expected a statement"); match stmt { - Statement::DefineTempTable { xml_options, fields, .. } => { + Statement::DefineTempTable { + xml_options, + fields, + .. + } => { assert_eq!(xml_options.namespace_uri.as_ref().unwrap().name, "urn:test"); assert_eq!(xml_options.serialize_name.as_ref().unwrap().name, "myTable"); assert_eq!(fields.len(), 1); @@ -7028,7 +7135,8 @@ fn parse_define_temp_table_xml_options() { #[test] fn parse_define_buffer_xml_options() { - let source = r#"DEFINE BUFFER bCust FOR Customer NAMESPACE-URI "urn:foo" SERIALIZE-NAME "cust"."#; + let source = + r#"DEFINE BUFFER bCust FOR Customer NAMESPACE-URI "urn:foo" SERIALIZE-NAME "cust"."#; let tokens = tokenize(source); let mut parser = Parser::new(&tokens, source); let stmt = parser.parse_statement().expect("Expected a statement"); diff --git a/resources/bench_parser_datasets.abl b/resources/bench_parser_datasets.abl new file mode 100644 index 0000000..5ef988e --- /dev/null +++ b/resources/bench_parser_datasets.abl @@ -0,0 +1,88 @@ +/* Benchmark fixture: Dataset and Data-Source definitions */ + +/* Simple dataset with one relation */ +DEFINE TEMP-TABLE ttCustomer NO-UNDO + FIELD CustNum AS INTEGER + FIELD Name AS CHARACTER. + +DEFINE TEMP-TABLE ttOrder NO-UNDO + FIELD OrderNum AS INTEGER + FIELD CustNum AS INTEGER + FIELD OrderDate AS DATE. + +DEFINE TEMP-TABLE ttOrderLine NO-UNDO + FIELD OrderNum AS INTEGER + FIELD LineNum AS INTEGER + FIELD ItemId AS INTEGER + FIELD Qty AS INTEGER. + +DEFINE DATASET dsCustomerOrders FOR ttCustomer, ttOrder, ttOrderLine + DATA-RELATION drCustOrder FOR ttCustomer, ttOrder + RELATION-FIELDS (CustNum, CustNum) + DATA-RELATION drOrderLine FOR ttOrder, ttOrderLine + RELATION-FIELDS (OrderNum, OrderNum). + +/* Dataset with all flags */ +DEFINE DATASET dsWithFlags FOR ttCustomer, ttOrder + DATA-RELATION FOR ttCustomer, ttOrder + RELATION-FIELDS (CustNum, CustNum) + REPOSITION NESTED FOREIGN-KEY-HIDDEN NOT-ACTIVE RECURSIVE. + +/* Dataset with XML/serialize options */ +DEFINE DATASET dsXml + NAMESPACE-URI "urn:example:customer" + NAMESPACE-PREFIX "cust" + XML-NODE-NAME "CustomerData" + SERIALIZE-NAME "customerDataset" + SERIALIZE-HIDDEN + FOR ttCustomer, ttOrder + DATA-RELATION FOR ttCustomer, ttOrder + RELATION-FIELDS (CustNum, CustNum). + +/* Dataset with REFERENCE-ONLY */ +DEFINE DATASET dsRef REFERENCE-ONLY FOR ttCustomer. + +/* Dataset with PARENT-ID-RELATION */ +DEFINE TEMP-TABLE ttParent NO-UNDO + FIELD ParentId AS INTEGER + FIELD ParentName AS CHARACTER. + +DEFINE TEMP-TABLE ttChild NO-UNDO + FIELD ChildId AS INTEGER + FIELD ParentRecid AS INTEGER + FIELD ChildName AS CHARACTER. + +DEFINE DATASET dsParentChild FOR ttParent, ttChild + PARENT-ID-RELATION pidRel FOR ttParent, ttChild + PARENT-ID-FIELD ParentRecid + PARENT-FIELDS-BEFORE (ParentId, ParentName) + PARENT-FIELDS-AFTER (ParentName). + +/* Dataset with access modifiers */ +DEFINE PRIVATE STATIC SERIALIZABLE DATASET dsPrivate FOR ttCustomer. + +/* NEW SHARED dataset */ +DEFINE NEW SHARED DATASET dsShared FOR ttCustomer. + +/* Data-source definitions */ +DEFINE DATA-SOURCE dsCust FOR Customer. + +DEFINE DATA-SOURCE dsMulti FOR Customer, Order. + +DEFINE DATA-SOURCE dsWithQuery FOR QUERY qCust Customer. + +DEFINE DATA-SOURCE dsWithKeys FOR Customer KEYS (CustNum, Region). + +DEFINE DATA-SOURCE dsWithRowid FOR Customer KEYS (ROWID). + +DEFINE PRIVATE STATIC DATA-SOURCE dsStatic FOR Customer. + +/* CREATE typed variants */ +DEFINE VARIABLE hDs AS HANDLE NO-UNDO. +DEFINE VARIABLE hSrc AS HANDLE NO-UNDO. +DEFINE VARIABLE hTt AS HANDLE NO-UNDO. + +CREATE DATASET hDs. +CREATE DATA-SOURCE hSrc. +CREATE TEMP-TABLE hTt. +CREATE Customer. From 7d52eb5e7c275c26a0a08530865592043ca7f30c Mon Sep 17 00:00:00 2001 From: Evan Robertson Date: Thu, 9 Apr 2026 19:18:39 -0400 Subject: [PATCH 5/7] docs: mark dataset plan as completed --- ...1-feat-dataset-data-source-support-plan.md | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/plans/2026-04-07-001-feat-dataset-data-source-support-plan.md b/docs/plans/2026-04-07-001-feat-dataset-data-source-support-plan.md index bdc2bbc..5d563e9 100644 --- a/docs/plans/2026-04-07-001-feat-dataset-data-source-support-plan.md +++ b/docs/plans/2026-04-07-001-feat-dataset-data-source-support-plan.md @@ -1,7 +1,7 @@ --- title: "feat: Add dataset and data-source parsing support" type: feat -status: active +status: completed date: 2026-04-07 origin: docs/brainstorms/2026-04-07-dataset-support-brainstorm.md --- @@ -544,17 +544,17 @@ Use `Option` for optional single values, `Vec` (empty = absent) for repeat ## Acceptance Criteria -- [ ] All new keywords added to `keyword_overrides.toml` and codegen runs cleanly -- [ ] `parse_define_dataset()` handles all documented clauses: modifiers, XML/serialize options, REFERENCE-ONLY, FOR buffers, DATA-RELATION (with RELATION-FIELDS, REPOSITION, NESTED, FOREIGN-KEY-HIDDEN, NOT-ACTIVE, RECURSIVE), PARENT-ID-RELATION (with PARENT-ID-FIELD, PARENT-FIELDS-BEFORE/AFTER) -- [ ] `parse_define_data_source()` handles QUERY and source-buffer-phrases with KEYS -- [ ] `parse_create_statement()` dispatches on DATASET, DATA-SOURCE, and TEMP-TABLE with optional IN WIDGET-POOL -- [ ] `parse_define_parameter()` handles TABLE, TABLE-HANDLE, DATASET, DATASET-HANDLE, and BUFFER parameter types -- [ ] DEFINE TEMP-TABLE and DEFINE BUFFER properly parse XML/serialize options (skip-unknown hack removed) -- [ ] All existing tests still pass (no regressions from AST restructuring) -- [ ] New tests cover all clause combinations listed in Phase 8 -- [ ] Benchmark fixture added and discoverable by CodSpeed -- [ ] `cargo clippy -D warnings` passes -- [ ] `cargo fmt --check` passes +- [x] All new keywords added to `keyword_overrides.toml` and codegen runs cleanly +- [x] `parse_define_dataset()` handles all documented clauses: modifiers, XML/serialize options, REFERENCE-ONLY, FOR buffers, DATA-RELATION (with RELATION-FIELDS, REPOSITION, NESTED, FOREIGN-KEY-HIDDEN, NOT-ACTIVE, RECURSIVE), PARENT-ID-RELATION (with PARENT-ID-FIELD, PARENT-FIELDS-BEFORE/AFTER) +- [x] `parse_define_data_source()` handles QUERY and source-buffer-phrases with KEYS +- [x] `parse_create_statement()` dispatches on DATASET, DATA-SOURCE, and TEMP-TABLE with optional IN WIDGET-POOL +- [x] `parse_define_parameter()` handles TABLE, TABLE-HANDLE, DATASET, DATASET-HANDLE, and BUFFER parameter types +- [x] DEFINE TEMP-TABLE and DEFINE BUFFER properly parse XML/serialize options (skip-unknown hack removed) +- [x] All existing tests still pass (no regressions from AST restructuring) +- [x] New tests cover all clause combinations listed in Phase 8 +- [x] Benchmark fixture added and discoverable by CodSpeed +- [x] `cargo clippy -D warnings` passes +- [x] `cargo fmt --check` passes ## Dependencies & Risks From 71a84063ac5bc2ce7db38c502e7b7b759ca87ba6 Mon Sep 17 00:00:00 2001 From: Evan Robertson Date: Thu, 9 Apr 2026 20:41:50 -0400 Subject: [PATCH 6/7] fix(parser): add new keywords to can_be_identifier Adding Kind::Nested and other dataset keywords caused them to no longer be recognized as valid identifiers in member access, method call, and other identifier positions. Add all new unreserved keywords to can_be_identifier() to fix chained member access parsing. --- crates/oxabl_parser/src/parser/mod.rs | 29 +++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/crates/oxabl_parser/src/parser/mod.rs b/crates/oxabl_parser/src/parser/mod.rs index ee5c37e..39f0957 100644 --- a/crates/oxabl_parser/src/parser/mod.rs +++ b/crates/oxabl_parser/src/parser/mod.rs @@ -242,6 +242,35 @@ impl<'a> Parser<'a> { | Kind::Clob | Kind::Blob | Kind::ComHandle + // Dataset / data-source keywords (unreserved) + | Kind::Dataset + | Kind::DatasetHandle + | Kind::DataRelation + | Kind::DataSource + | Kind::NamespaceUri + | Kind::NamespacePrefix + | Kind::XmlNodeName + | Kind::XmlNodeType + | Kind::SerializeName + | Kind::SerializeHidden + | Kind::Serializable + | Kind::NonSerializable + | Kind::ReferenceOnly + | Kind::RelationFields + | Kind::Nested + | Kind::ForeignKeyHidden + | Kind::NotActive + | Kind::Recursive + | Kind::ParentIdRelation + | Kind::ParentIdField + | Kind::ParentFieldsBefore + | Kind::ParentFieldsAfter + | Kind::WidgetPool + | Kind::TableHandle + | Kind::Bind + | Kind::ByValue + | Kind::Query + | Kind::Reposition ) } From 12b4041b1c3f6dd9de016fcdcd1cb4d2caeed6f3 Mon Sep 17 00:00:00 2001 From: Evan Robertson Date: Thu, 9 Apr 2026 20:52:33 -0400 Subject: [PATCH 7/7] chore(tests): removed unused imports --- crates/oxabl_parser/src/parser/tests.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/crates/oxabl_parser/src/parser/tests.rs b/crates/oxabl_parser/src/parser/tests.rs index 9500e93..5151d97 100644 --- a/crates/oxabl_parser/src/parser/tests.rs +++ b/crates/oxabl_parser/src/parser/tests.rs @@ -1,10 +1,9 @@ use super::*; use oxabl_ast::{ - AccessModifier, BooleanLiteral, BufferTarget, CreateTarget, CreateTargetKind, DataRelation, - DataSourceBuffer, DataSourceKeys, DataType, DecimalLiteral, Expression, FieldTypeSource, - FindType, HandleParamKind, HandlePassingOptions, Identifier, IntegerLiteral, Literal, LockType, - ParameterDirection, ParameterType, RunTarget, SortDirection, Span, Statement, StreamDirection, - StreamOperation, StringLiteral, UnknownLiteral, WhenBranch, XmlSerializeOptions, + AccessModifier, BooleanLiteral, BufferTarget, CreateTarget, CreateTargetKind, DataSourceKeys, + DataType, DecimalLiteral, Expression, FieldTypeSource, FindType, HandleParamKind, Identifier, + IntegerLiteral, Literal, LockType, ParameterDirection, ParameterType, RunTarget, SortDirection, + Span, Statement, StreamDirection, StreamOperation, StringLiteral, UnknownLiteral, WhenBranch, }; use oxabl_lexer::tokenize; use rust_decimal::Decimal;