From b5bc14b1fe661e5c01a123b52ed848fdee11ff57 Mon Sep 17 00:00:00 2001 From: megakuo Date: Thu, 31 Oct 2024 21:59:09 +0000 Subject: [PATCH 1/3] ast string support --- .../dsp/SimpleBlocks/include/toy/AST.h | 15 +++++++++++++ .../dsp/SimpleBlocks/include/toy/Lexer.h | 20 ++++++++++++++++++ .../dsp/SimpleBlocks/include/toy/Parser.h | 21 +++++++++++++++++-- mlir/examples/dsp/SimpleBlocks/parser/AST.cpp | 10 ++++++++- mlir/test/Examples/DspExample/dsp_string.py | 9 ++++++++ 5 files changed, 72 insertions(+), 3 deletions(-) create mode 100644 mlir/test/Examples/DspExample/dsp_string.py diff --git a/mlir/examples/dsp/SimpleBlocks/include/toy/AST.h b/mlir/examples/dsp/SimpleBlocks/include/toy/AST.h index c13b287bdd2f..d6b6f2c0e50e 100644 --- a/mlir/examples/dsp/SimpleBlocks/include/toy/AST.h +++ b/mlir/examples/dsp/SimpleBlocks/include/toy/AST.h @@ -43,6 +43,7 @@ class ExprAST { Expr_BinOp, Expr_Call, Expr_Print, + Expr_String, }; ExprAST(ExprASTKind kind, Location location) @@ -107,6 +108,20 @@ class VariableExprAST : public ExprAST { static bool classof(const ExprAST *c) { return c->getKind() == Expr_Var; } }; +/// Expression class for string val. +class StringExprAST : public ExprAST { + std::string string_val; + +public: + StringExprAST(Location loc, llvm::StringRef string_val) + : ExprAST(Expr_String, std::move(loc)), string_val(string_val) {} + + llvm::StringRef getStringVal() { return string_val; } + + /// LLVM style RTTI + static bool classof(const ExprAST *c) { return c->getKind() == Expr_String; } +}; + /// Expression class for defining a variable. class VarDeclExprAST : public ExprAST { std::string name; diff --git a/mlir/examples/dsp/SimpleBlocks/include/toy/Lexer.h b/mlir/examples/dsp/SimpleBlocks/include/toy/Lexer.h index 38678e23f553..4c7849f7d8a1 100644 --- a/mlir/examples/dsp/SimpleBlocks/include/toy/Lexer.h +++ b/mlir/examples/dsp/SimpleBlocks/include/toy/Lexer.h @@ -44,6 +44,7 @@ enum Token : int { tok_return = -2, tok_var = -3, tok_def = -4, + tok_string_val = -7, // primary tok_identifier = -5, @@ -84,6 +85,11 @@ class Lexer { return identifierStr; } + llvm::StringRef getString() { + assert(curTok == tok_string_val); + return stringVal; + } + /// Return the current number (prereq: getCurToken() == tok_number) double getValue() { assert(curTok == tok_number); @@ -173,6 +179,17 @@ class Lexer { return getTok(); } + // String val: "..." + if(lastChar == '"') { + stringVal = ""; + while (isalnum((lastChar = Token(getNextChar()))) || lastChar == '_') { + if(lastChar == '"') break; + stringVal += (char)lastChar; + } + lastChar = Token(getNextChar()); + return tok_string_val; + } + // Check for end of file. Don't eat the EOF. if (lastChar == EOF) return tok_eof; @@ -191,6 +208,9 @@ class Lexer { /// If the current Token is an identifier, this string contains the value. std::string identifierStr; + + // If current Token is a string val + std::string stringVal; /// If the current Token is a number, this contains the value. double numVal = 0; diff --git a/mlir/examples/dsp/SimpleBlocks/include/toy/Parser.h b/mlir/examples/dsp/SimpleBlocks/include/toy/Parser.h index 42bd653b156c..238f7d441676 100644 --- a/mlir/examples/dsp/SimpleBlocks/include/toy/Parser.h +++ b/mlir/examples/dsp/SimpleBlocks/include/toy/Parser.h @@ -167,6 +167,16 @@ class Parser { return v; } + /// parenexpr ::= '"' string_val '"' + std::unique_ptr parseStringExpr() { + auto loc = lexer.getLastLocation(); + + std::string string_val(lexer.getString()); + lexer.consume(tok_string_val); + + return std::make_unique(std::move(loc), string_val); + } + /// identifierexpr /// ::= identifier /// ::= identifier '(' expression ')' @@ -175,7 +185,7 @@ class Parser { auto loc = lexer.getLastLocation(); lexer.getNextToken(); // eat identifier. - + if (lexer.getCurToken() != '(') // Simple variable ref. return std::make_unique(std::move(loc), name); @@ -216,6 +226,7 @@ class Parser { /// ::= numberexpr /// ::= parenexpr /// ::= tensorliteral + /// ::= stringexpr std::unique_ptr parsePrimary() { switch (lexer.getCurToken()) { default: @@ -230,6 +241,8 @@ class Parser { return parseParenExpr(); case '[': return parseTensorLiteralExpr(); + case tok_string_val: + return parseStringExpr(); case ';': return nullptr; case '}': @@ -334,7 +347,11 @@ class Parser { if (!type) type = std::make_unique(); lexer.consume(Token('=')); - auto expr = parseExpression(); + std::unique_ptr expr; + if(lexer.getCurToken() == tok_string_val) { + expr = parseStringExpr(); + } + else expr = parseExpression(); return std::make_unique(std::move(loc), std::move(id), std::move(*type), std::move(expr)); } diff --git a/mlir/examples/dsp/SimpleBlocks/parser/AST.cpp b/mlir/examples/dsp/SimpleBlocks/parser/AST.cpp index a5dfa2d0f16e..6b824aa59997 100644 --- a/mlir/examples/dsp/SimpleBlocks/parser/AST.cpp +++ b/mlir/examples/dsp/SimpleBlocks/parser/AST.cpp @@ -51,6 +51,7 @@ class ASTDumper { void dump(PrintExprAST *node); void dump(PrototypeAST *node); void dump(FunctionAST *node); + void dump(StringExprAST *node); // Actually print spaces matching the current indentation level void indent() { @@ -81,7 +82,7 @@ static std::string loc(T *node) { void ASTDumper::dump(ExprAST *expr) { llvm::TypeSwitch(expr) .Case( + PrintExprAST, ReturnExprAST, VarDeclExprAST, StringExprAST, VariableExprAST>( [&](auto *node) { this->dump(node); }) .Default([&](ExprAST *) { // No match, fallback to a generic message @@ -90,6 +91,13 @@ void ASTDumper::dump(ExprAST *expr) { }); } +/// A string expression +void ASTDumper::dump(StringExprAST *stringExpr) { + INDENT(); + llvm::errs() << "StringExpr \"" << stringExpr->getStringVal() << "\""; + llvm::errs() << " " << loc(stringExpr) << "\n"; +} + /// A variable declaration is printing the variable name, the type, and then /// recurse in the initializer value. void ASTDumper::dump(VarDeclExprAST *varDecl) { diff --git a/mlir/test/Examples/DspExample/dsp_string.py b/mlir/test/Examples/DspExample/dsp_string.py new file mode 100644 index 000000000000..1523981dbada --- /dev/null +++ b/mlir/test/Examples/DspExample/dsp_string.py @@ -0,0 +1,9 @@ +def main() { + var a = [[[10,20],[30,0]] ]; + var b = [[[40,50],[60,70]] ]; + var c = sub(a, b); + print(c); + + var d = "12345"; + print(d); +} From 64f7da73695a9a934e96ec24fa0311cd805c646f Mon Sep 17 00:00:00 2001 From: megakuo Date: Thu, 31 Oct 2024 22:45:53 +0000 Subject: [PATCH 2/3] support string type in dsp mlir --- .../dsp/SimpleBlocks/include/toy/Lexer.h | 2 +- .../dsp/SimpleBlocks/mlir/MLIRGen.cpp | 25 +++++++++++++++++++ mlir/test/Examples/DspExample/dsp_string.py | 2 +- 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/mlir/examples/dsp/SimpleBlocks/include/toy/Lexer.h b/mlir/examples/dsp/SimpleBlocks/include/toy/Lexer.h index 4c7849f7d8a1..3d5827638470 100644 --- a/mlir/examples/dsp/SimpleBlocks/include/toy/Lexer.h +++ b/mlir/examples/dsp/SimpleBlocks/include/toy/Lexer.h @@ -182,7 +182,7 @@ class Lexer { // String val: "..." if(lastChar == '"') { stringVal = ""; - while (isalnum((lastChar = Token(getNextChar()))) || lastChar == '_') { + while (isalnum((lastChar = Token(getNextChar()))) || lastChar == '_' || lastChar== ' ') { if(lastChar == '"') break; stringVal += (char)lastChar; } diff --git a/mlir/examples/dsp/SimpleBlocks/mlir/MLIRGen.cpp b/mlir/examples/dsp/SimpleBlocks/mlir/MLIRGen.cpp index ff56ef6d95fa..90be46515a6b 100644 --- a/mlir/examples/dsp/SimpleBlocks/mlir/MLIRGen.cpp +++ b/mlir/examples/dsp/SimpleBlocks/mlir/MLIRGen.cpp @@ -39,6 +39,7 @@ #include #include #include +#include using namespace mlir::dsp; using namespace dsp; @@ -947,6 +948,28 @@ class MLIRGenImpl { mlir::Value mlirGen(NumberExprAST &num) { return builder.create(loc(num.loc()), num.getValue()); } + + /// Emit a string exression + mlir::Value mlirGen(StringExprAST &expr) { + auto string_val = expr.getStringVal(); + + std::vector signals; + for(char ch : string_val) { + std::bitset<8> bits(static_cast(ch)), reversed; + int n = 8; + for(int i=0; i(loc(expr.loc()), type, dataAttr); + } /// Dispatch codegen for the right expression subclass using RTTI. mlir::Value mlirGen(ExprAST &expr) { @@ -961,6 +984,8 @@ class MLIRGenImpl { return mlirGen(cast(expr)); case dsp::ExprAST::Expr_Num: return mlirGen(cast(expr)); + case dsp::ExprAST::Expr_String: + return mlirGen(cast(expr)); default: emitError(loc(expr.loc())) << "MLIR codegen encountered an unhandled expr kind '" diff --git a/mlir/test/Examples/DspExample/dsp_string.py b/mlir/test/Examples/DspExample/dsp_string.py index 1523981dbada..7734defb9307 100644 --- a/mlir/test/Examples/DspExample/dsp_string.py +++ b/mlir/test/Examples/DspExample/dsp_string.py @@ -4,6 +4,6 @@ def main() { var c = sub(a, b); print(c); - var d = "12345"; + var d = "HELLO FROM SPACE"; print(d); } From 50d0a505f931864e37e869db9bd9b82623c006a0 Mon Sep 17 00:00:00 2001 From: megakuo Date: Fri, 1 Nov 2024 00:08:15 +0000 Subject: [PATCH 3/3] add test case --- mlir/test/Examples/DspExample/dsp_string.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mlir/test/Examples/DspExample/dsp_string.py b/mlir/test/Examples/DspExample/dsp_string.py index 7734defb9307..ead95cf07e73 100644 --- a/mlir/test/Examples/DspExample/dsp_string.py +++ b/mlir/test/Examples/DspExample/dsp_string.py @@ -6,4 +6,5 @@ def main() { var d = "HELLO FROM SPACE"; print(d); + print("abd"); }