diff --git a/.github/workflows/build-documentation.yml b/.github/workflows/build-documentation.yml new file mode 100644 index 0000000..c746f95 --- /dev/null +++ b/.github/workflows/build-documentation.yml @@ -0,0 +1,58 @@ +name: Build documentation + +on: + push: + workflow_dispatch: + +env: + INSTANCE: 'Writerside/strumenta' + DOCKER_VERSION: '243.21565' + +jobs: + build: + runs-on: ubuntu-latest + outputs: + algolia_artifact: ${{ steps.define-ids.outputs.algolia_artifact }} + artifact: ${{ steps.define-ids.outputs.artifact }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Define instance id and artifacts + id: define-ids + run: | + INSTANCE=${INSTANCE#*/} + INSTANCE_ID_UPPER=$(echo "$INSTANCE" | tr '[:lower:]' '[:upper:]') + ARTIFACT="starlasu-documentation-${INSTANCE_ID_UPPER}.zip" + ALGOLIA_ARTIFACT="algolia-indexes-${INSTANCE_ID_UPPER}.zip" + + # Print the values + echo "INSTANCE_ID_UPPER: $INSTANCE_ID_UPPER" + echo "ARTIFACT: $ARTIFACT" + echo "ALGOLIA_ARTIFACT: $ALGOLIA_ARTIFACT" + + # Set the environment variables and outputs + echo "INSTANCE_ID_UPPER=$INSTANCE_ID_UPPER" >> $GITHUB_ENV + echo "ARTIFACT=$ARTIFACT" >> $GITHUB_ENV + echo "ALGOLIA_ARTIFACT=$ALGOLIA_ARTIFACT" >> $GITHUB_ENV + echo "artifact=$ARTIFACT" >> $GITHUB_OUTPUT + echo "algolia_artifact=$ALGOLIA_ARTIFACT" >> $GITHUB_OUTPUT + + - name: Build docs using Writerside Docker builder + uses: JetBrains/writerside-github-action@v4 + with: + instance: ${{ env.INSTANCE }} + artifact: ${{ env.ARTIFACT }} + docker-version: ${{ env.DOCKER_VERSION }} + + - name: Save artifact with build results + uses: actions/upload-artifact@v4 + with: + name: docs + path: | + artifacts/${{ env.ARTIFACT }} + artifacts/report.json + artifacts/${{ env.ALGOLIA_ARTIFACT }} + retention-days: 7 \ No newline at end of file diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml deleted file mode 100644 index e051b70..0000000 --- a/.github/workflows/package.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: Publish Generated Documentation - -on: - push: - branches: [ main ] - -jobs: - - archive-build-artifacts: - runs-on: ubuntu-latest - steps: - - - uses: actions/checkout@v2 - - - name: Prepare documentation generation - run: sh preparedoc.sh - - - name: Generate HTML documentation - uses: docker://pandoc/latex:2.9 - with: - args: -c css/documentation.css -s build/source.md --metadata pagetitle="StarLasu Overview" -o build/htmldoc/StarLasuOverview.html - - - name: Generate PDF documentation - uses: docker://pandoc/latex:2.9 - with: - args: -c css/documentation.css -H configuration/listings-setup.tex --variable colorlinks=true build/source.md --listings --metadata pagetitle="StarLasu Overview" -o build/pdfdoc/StarLasuOverview.pdf - - - name: Upload HTML documentation - uses: actions/upload-artifact@v2 - with: - name: StarLasu Documentation (HTML) - path: build/htmldoc - - - name: Upload PDF documentation - uses: actions/upload-artifact@v2 - with: - name: StarLasu Documentation (PDF) - path: build/pdfdoc diff --git a/README.md b/README.md index 3ca88c7..36b25ea 100644 --- a/README.md +++ b/README.md @@ -1,24 +1,9 @@ -# StarLasu -StarLasu is the missing link between source code and a convenient structure for its interpretation and manipulation: an AST. -When building an interpreter, transpiler, compiler, editor, static analysis tool, etc., at Strumenta we always implement the following pipeline: - -Source code --ANTLR4 parser--> Parse tree __--StarLasu--> AST__ --Further processing--> ... --> Result - -StarLasu is both the above methodology and a collection of runtime libraries to support it in Java, Kotlin, Python, Javascript, Typescript, and C#: - -* [Kolasu](https://github.com/Strumenta/kolasu): it was the first library to be developed. It supports development in Kotlin and Java. -* [Tylasu](https://github.com/Strumenta/tylasu): it supports development in Typescript and Javascript. We're working to advance it to be on par with Kolasu. -* [Pylasu](https://github.com/Strumenta/pylasu): it supports development in Python. It is fairly recent but it is progressing quite rapidly. -* [Sharplasu](https://github.com/Strumenta/sharplasu): it supports development in C#. It is the most recently created and it is ongoing active development. - -## Documentation - -Documentation for StarLasu is available [here](https://github.com/Strumenta/StarLasu/tree/main/documentation). +# Starlasu +This project contains the documentation on the Starlasu approach, which is +published [here](https://github.com/Strumenta/StarLasu/tree/main/documentation). ## Publication -You will need pandoc with pdflatex (`brew install pandoc basictex`). +We generate the documentation to be published using JetBrains's Writerside. Updating the published version requires manually uploading the generated files to Netlify. At the moment Federico is the only one who has access. - -You will need to run generatedoc.sh, then take the content of htmldoc and renamed the html file to index.html, zip the html file with the css directory and upload it to netlify. diff --git a/Writerside/c.list b/Writerside/c.list new file mode 100644 index 0000000..c4c77a2 --- /dev/null +++ b/Writerside/c.list @@ -0,0 +1,6 @@ + + + + + \ No newline at end of file diff --git a/Writerside/cfg/buildprofiles.xml b/Writerside/cfg/buildprofiles.xml new file mode 100644 index 0000000..5e6a73d --- /dev/null +++ b/Writerside/cfg/buildprofiles.xml @@ -0,0 +1,13 @@ + + + + + + + + false + + + + diff --git a/Writerside/images/Star.svg b/Writerside/images/Star.svg new file mode 100644 index 0000000..af261c2 --- /dev/null +++ b/Writerside/images/Star.svg @@ -0,0 +1,8 @@ + + + + + + diff --git a/Writerside/images/StarlasuOverview.png b/Writerside/images/StarlasuOverview.png new file mode 100644 index 0000000..fca3ff7 Binary files /dev/null and b/Writerside/images/StarlasuOverview.png differ diff --git a/Writerside/strumenta.tree b/Writerside/strumenta.tree new file mode 100644 index 0000000..1189a84 --- /dev/null +++ b/Writerside/strumenta.tree @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Writerside/topics/ASTRepresentation.md b/Writerside/topics/ASTRepresentation.md new file mode 100644 index 0000000..22e4f43 --- /dev/null +++ b/Writerside/topics/ASTRepresentation.md @@ -0,0 +1,647 @@ +# AST Representation + +Starlasu operates on Abstract Syntax Trees (ASTs) (or _Code Models_). These are tree-like data structures to represent +the information contained in a piece of formal language or “code”. For example, the statements in a procedural program, +the data elements in a SQL query, or the steps in a business workflow. + +All the Starlasu ASTs are based on a few primitive elements. + +The structure is similar to the one used by other Modeling solutions such as [EMF](https://eclipse.dev/modeling/emf/), +[MPS](https://www.jetbrains.com/mps/), or [LionWeb](https://lionweb.io/). +In particular we aim to (mostly) converge to the same terminology used in LionWeb +(see the [LioWeb’s specifications](https://lionweb.io/specification/metametamodel/metametamodel.html)). + +This is a representation of the whole structure. + +```mermaid +--- +title: Starlasu M3 +--- +classDiagram + +class Language { + qualifiedName: String + simpleName: String +} +Language "1" *--> "0..*" Type: types + +class Type { + <> + name: String +} +Type <|-- Classifier +Type <|-- DataType + +%% Concepts + +class Classifier { + <> +} +Classifier "1" *--> "0..*" Feature: features +Classifier <|-- Concept +Classifier <|-- ConceptInterface +Classifier <|-- Annotation + +class Concept +Concept "1" --> "0..1" Concept: extends +Concept "1" --> "0..*" ConceptInterface: implements + +class ConceptInterface +ConceptInterface "1" --> "0..*" ConceptInterface: extends + +%% Features + +class Feature { + <> + name: String + multiplicity: Multiplicity +} +Feature <|-- Property +Feature <|-- Link +Feature "1" --> "1" Type: type + +class Property { + type: DataType +} +class Link { + <> + type: Classifier +} +Link <|-- Containment +Link <|-- Reference + +class Containment +class Reference + +class Annotation { + multiple: Boolean +} +Annotation "1" --> "0..1" Annotation: extends +Annotation "1" --> "0..*" ConceptInterface: implements +Annotation "1" --> "0..1" Classifier: annotates + +%% DataTypes + +class DataType { + <> +} + +DataType <|-- PrimitiveType +class PrimitiveType + +DataType <|-- EnumType +class EnumType +EnumType "1" *--> "0..*" EnumerationLiteral: literals + +class EnumerationLiteral { + name: String +} + +%% Supporting enumerations + +class Multiplicity { + <> + SINGULAR + OPTIONAL + MANY +} + +``` + +***Note that this is the terminology used since Kolasu 1.6. In previous versions of Kolasu and other libraries +the terminology could be slightly different.*** + +We call this structure the Meta-Metamodel of Starlasu or the Starlasu’s M3 +(see https://en.wikipedia.org/wiki/Meta-Object_Facility for a definition of M0, M1, M2, and M3). + +In our case: + +- The M0 or the observed element is the code itself +- The M1 or the model of the code, is an instance of an AST. For example, an actual instance of ClassDeclaration with name *foo*, no fields and no methods +- The M2 or the metamodel is the set of AST classes defined for a certain language, for example the definition of the class ClassDeclaration, describing the fact that a ClassDeclaration has a name, can extend zero or one other ClassDeclarations and can implement zero to many InterfaceDeclarations +- The M3 or the meta-metamodel is the structure that we use to define AST. See the picture above + +## Language + +- At the top level there is a *Language.* See for example, [KolasuLanguage](https://github.com/Strumenta/kolasu/blob/2fa276186924ec859519d3c87fad6513c23fd74a/core/src/main/kotlin/com/strumenta/kolasu/language/KolasuLanguage.kt#L22) +- A *Language* has a qualified name. It has also a simple name, which is simply the portion of the qualified name following the last dot (or the entire qualified name, if it has no dots) +- A *Language* groups various *Types.* +- *Types* can be either *Concepts, ConceptInterfaces, PrimitiveTypes,* or *EnumTypes* + +## Concept, ConceptInterface, Annotation + +- A *Concept* can extend zero or one *Concepts*. It can implement zero to many *ConceptInterfaces* +- *Concepts* and *ConceptInterfaces* have a name (because they are *Types*) +- *Concepts* and *ConceptInterfaces* have features +- *ConceptInterfaces* are similar to interfaces in programming languages +- An *Annotation* can extend another *Annotation*. It can implement zero many *ConceptInterfaces* + +## Features + +- All features have a name and could be either *Properties* or *Links* +- *Multiplicity* is one of: optional (zero or one instances), singular (exactly one instance), or many (zero to many instances). There is not alternative for one or more instances +- *Properties* have a type which is a *DataType*. *Properties* can have multiplicity singular or optional, but not many +- A *Link* is either a *Reference* or a *Containment* +- A *Link* has as type a *Classifier* (either a *Concept* or a *ConceptInterface*) +- A *Reference* can have multiplicity singular or optional, but not many + +## Primitive Types + +- A *Primitive Type* is anything that is not a *Node* is considered a primitive type. Typically these are Strings, Chars, Ints. This does not include Enums, which are treated separately. Arbitrary other types can be used. They may require additional configuration for example, for serialization. + + For example, consider this class from the RPG Language Module: + + ```kotlin + // Class _not_ extending Node + sealed class EditCode { + ... + } + + data class OutputSpecificationFieldDescription( + val fieldIndicators: List = emptyList(), + val fieldName: FieldNameType, + // EditCode is used in an AST and it is not a Node neither a + // ReferenceValue so it is a PrimitiveType. + // editCode is a Property, as the value is a PrimitiveType + val editCode: EditCode? = null, + val blankAfter: Boolean = false, + val endPosition: EndPosition? = null, + val dataFormat: DataFormat = DataFormat.Blank, + val various: Expression? = null, + val comments: String? = null + ) : OutputSpecification(), WithInlineDeclaration + ``` + + +## Enum Types + +These are just types representing a finite set of possibilities. + +## Obtaining Concepts from classes + +*Concepts* are automatically derived from inspecting classes. + +In the case of Kolasu, we inspect all the properties (in the “Kotlin sense” of property) of a Class, +ignoring the one marked as `@Internal`. + +For each property: + +- If the type is `ReferenceValue` (previously called `ReferenceByName`), then it indicates a *Reference* +- If the type is a Node or a Node subclass, then it indicates a *Containment* +- If the type a MutableList of Node or a Node subclass, then it indicates a *Containment* +- If the type is anything else, then we have a *Property* + +We want to keep this structure as minimal as possible, because in this way it is much easier to support more features such as serialization, transformers, interoperability with LionWeb, etc. + +For this reason we do not support sets or maps or other collections. + +For example: + +```kotlin +// This will produce a Concept named JCompilationUnit +data class JCompilationUnit( + // This will produce a containment with multiplicity Optional + val packageDeclaration: JPackageDeclaration? = null, + // This will produce a containment with multiplicity Many + val imports: MutableList = mutableListOf(), + // This will produce a containment with multiplicity Many + val declarations: MutableList = mutableListOf(), + // This will produce a containment with multiplicity Optional + val moduleDeclaration: JModuleDeclaration? = null, +) : Node() + +// This will produce a Concept extending the Concept derived +// from class JAnnotation +data class JSingleElementAnnotation( + // This will produce a reference with multiplicity Singular + override val typeName: ReferenceValue, + // This will produce a containment with multiplicity Singular + val value: JExpression, +) : JAnnotation(typeName) +``` + +# Guidelines on defining ASTs using Kolasu + +Now that we understand the Starlasu M3 and we know how it is derived from the Kotlin classes we create, we can see +how we should write our Kotlin classes to get the M2 (or metamodel or conceptual language) that we want. + +We typically specify attributes and single or optional containments with `var` . + +We typically specify references and multiple containments with `val` . In that case we can change the content of +the reference or the list, without the need to reassign them. + +## Annotations + +The internal annotation can be used to indicate fields that should not appear in the AST + +```kotlin +data class RPGPlaceholderExpr(@property:Internal override var placeholderName: String? = null) : + Expression(), PlaceholderElement { + @property:Internal + override val multiplePlaceholderElement: Boolean + get() = false +} +``` + +The derived annotation can be used to indicate features that are obtained by processing other features, and that +can be recalculated from them. + +```kotlin +data class CompilationUnit( + val mainStatements: MutableList = LinkedList(), + ... +) : Node(), StatementContainer { + ... + + @Derived + val mainRoutine: Subroutine + get() { + return Subroutine("VirtualMainRoutine", null, mainStatements, SubroutineCategory.Main) + .apply { + this.origin = + SimpleOrigin(mainStatements.firstOrNull()?.position, "").addToOriginNodes(mainStatements) + }.withParent(this) + } + +``` + +## Prefix + +We typically want to use a prefix for **all** the classes we use to define our language M2. This is useful because +in transpiler and other applications we may want to combine multiple languages. For example, in a transpiler from RPG to Java we may want to include both the RPG AST classes and the Java AST classes. Both languages could share constructs with the same name, and without a prefix it would be cumbersome to distinguish them. + +## Common Concepts + +Starlasu includes a few Common *ConceptInterfaces.* They can be used as markers, so that different tooling can look +for *Concepts* implementing those, and treated them in a special way. + +They are: + +- **Statement:** it represents a statement in the language (i.e., something that can be executed) +- **Expression:** it represents an expression in the language (i.e., something that can be evaluated and produce a value +- **EntityDeclaration:** it should be used for the definition of classes, interfaces, and structures. These declarations introduces new types. This interface should not be used for parameter declarations or variable declarations. +- **PlaceholderElement:** this indicates special *Concepts* used to represent variability when defining patterns. If you are not building support for patterns in your language, you can ignore them (see [Placeholders](https://www.notion.so/Placeholders-34efa5c5c3df43299ce87bef074b06be?pvs=21)) + +## How to use Named and PossiblyNamed + +*Named* is to be used for things that have **always** a name, *PossiblyNamed* is to be used for things that **may or may not** have name. + +For example, in some languages one can define functions and anonymous functions. If we want to represent both with the same Concept, then we can make that Concept implement *PossiblyNamed*. + +For a local variable which is guaranteed to have a name (supposing our language do not support anonymous variables) then we would use *Named*. + +## When to use ReferenceValue + +A *ReferenceByName* indicates that a name can indicate something else, where something else is typically another node in the same AST or another AST or a reference to some external symbol (like a builtin-function). + +Examples of usages of *ReferenceByName*: + +```kotlin +data class FReferenceExpr(val value: ReferenceByName) + : FExpression + +data class FMethodCallExpr(val method: ReferenceByName, + val args: MutableList = mutableListOf()) + : FExpression +``` + +## When to use enums + +Enums should be used when we have a set of alternatives. For example: + +```kotlin +enum class JVisibility { + PUBLIC, + PRIVATE, + PROTECTED, + DEFAULT +} + +sealed class JClassMember(val visibility: JVisibility): Node() +``` + +## Modifiers + +When representing modifiers we have two alternatives: + +- Represent the list of modifiers itself. We tend to do that to preserve the information on the order of the modifiers. This is a lower level representation, more suited for the parse tree than the AST. We should avoid it. +- To represent the modifiers through flags and enums in the declaration to which the modifiers are applied. This is closer to the mental model a developer would have of the code, and this is what we should do. + +```kotlin +// To use +data class FMethodDeclaration(val visibility: FVisibility, + val isFinal: Boolean, val isStatic: Boolean, + val name: String, val body: FBody? = null) : FMember() + +// To avoid +data class FMethodDeclaration(val modifiers: MutableList, + val name: String, val body: FBody? = null) : FMember() +``` + +## Statements + +We typically define a top level rule to represent all statements of our language: + +```kotlin +sealed class FStatement : BaseNode(), Statement +// here Expression is the common concept we have seen above +``` + +All other statements should end with the `Stmt` postfix. + +### Expression Statements + +In most languages methods or other blocks contain list of statements. They cannot contain directly expressions. However expressions can be wrapped in *ExpressionStatements*: + +```kotlin +data class FExpressionStmt(val expr: FExpression) : FStatement +``` + +For example, in Java we may have: + +```kotlin +public void foo() { + // here this entire line, including the semicolon, is an + // JExpressionStmt, while the expression 1 + 2, is a JAdditionExpr + 1 + 2; +} +``` + +## Expressions + +We typically define a top level rule to represent all expressions of our language: + +```kotlin +sealed class FExpression : BaseNode(), Expression +// here Expression is the common concept we have seen above +``` + +All other expressions should end with the `Expr` postfix. + +### Binary expressions + +```kotlin +sealed class FBinaryExpr( + open var left: FExpression, + open var right: FExpression) +: FExpression() + +// We should NOT do this +// The type of operations should be distinguished through subclasses +// We may well have an operator field in the parse tree, but we should +// not have in the AST +class FBinaryExpr( + open var left: FExpression, + open var right: FExpression, + val operator: OperatorType) +: FExpression() +``` + +### Arithmetic expressions + +```kotlin +sealed class FArithmeticExpr( + open var left: FExpression, + open var right: FExpression) +: FExpression() + +// do **not** call this FSumExpr or FAddExpr +data class FAdditionExpr( + override var left: FExpression, + override var right: FExpression) +: FArithmeticExpr() + +data class FSubtractionExpr( + override var left: FExpression, + override var right: FExpression) +: FArithmeticExpr() + +data class FMultiplicationExpr( + override var left: FExpression, + override var right: FExpression) +: FArithmeticExpr() + +data class FDivisionExpr( + override var left: FExpression, + override var right: FExpression) +: FArithmeticExpr() +``` + +### Comparison expressions + +```kotlin +sealed class FComparisonExpr( + override var left: FExpression, + override var right: FExpression) : + FBinaryExpr(left, right) + +data class FEqualityExpr( + override var left: FExpression, + override var right: FExpression) : + FComparisonExpr(left, right) + +// do **not** call this FDisequalityExpr +data class FInequalityExpr( + override var left: FExpression, + override var right: FExpression) : + FComparisonExpr(left, right) + +data class FLessThanExpr( + override var left: FExpression, + override var right: FExpression) : + FComparisonExpr(left, right) + +data class FGreaterThanExpression( + override var left: FExpression, + override var right: FExpression) : + FComparisonExpression(left, right) + +data class FLessOrEqualToExpression( + override var left: FExpression, + override var right: FExpression) : + FComparisonExpression(left, right) + +data class FGreaterOrEqualToExpr( + override var left: FExpression, + override var right: FExpression) : + FComparisonExpr(left, right) + +``` + +### Logical expressions + +```kotlin +// even if the language does not support bitwise and/or expressions +// these name make more obvious what these expressions are +data class FLogicalAndExpr( + override var left: FExpression, + override var right: FExpression) : + FBinaryExpr(left, right) + +data class FLogicalOrExpr( + override var left: FExpression, + override var right: FExpression) : + FBinaryExpr(left, right) +``` + +### Field accesses + +Field accesses are typically represented like this: + +```kotlin +data class FFieldAccessExpr(val container: FExpression, + val field: ReferenceValue) : FExpression() +``` + +Note that in case of chain of field accesses we would have a recurring structure. + +We should **not** aim to represent chains of field accesses as a single expression: + +```kotlin +// Do **not** do this +data class FFieldAccessExpr(val parts: List): FExpression() + +// Or this, which is not supported by Starlasu as lists of references +// are forbidden +data class FFieldAccessExpr( + val parts: List): FExpression() + +// Also this has to be avoided +data class FFieldAccessExpr( + val parts: List): FExpression() +data class FFieldAccessExprPart( + val field: ReferenceValue): Node() +``` + +Note that we discuss only *qualified* field accesses. So field accesses in the form: + +```java +container.field +``` + +We do not consider *unqualified* field accesses. The reason is that unqualified field accesses are undistinguishable from references to other expressions during parsing. We can distinguish them only during symbol resolution. For this reason unqualified field accesses are represented using reference expressions. + +### Function and Method calls + +Function and method calls are typically expressions, as they can produce a value: + +```kotlin +data class FMethodCallExpr( + val method: ReferenceValue, + val args: MutableList = mutableLisOf()) + : FExpression() +``` + +These represents method calls where the receiver (the thing on which the method is invoked) is not specified. + +```kotlin +myMethod() // simple or unqualified method call, see this section +myReceived.myMethod() // qualified method call, see next section +``` + +If both qualified and unqualified method calls are possible, a single abstract parent class should be used. It should contain the `method` and `args` parameters specified here. + +### Qualified method calls + +These represents method calls where the receiver (the thing on which the method is invoked) is specified. + +```kotlin +myMethod() // simple or unqualified method call, see previous section +myReceived.myMethod() // qualified method call, see this section +``` + +This is how they can be represented: + +```kotlin +data class FQualifiedMethodCallExpr( + var container: FExpression, + override val method: ReferenceValue, + override val args: MutableList = mutableLisOf()) + : FMethodCall() +``` + +We discourage having a single AST class to represent both unqualified and qualified method calls, by making the container optional. + +### Unary expressions + +```kotlin +data class FNegatedExpr(val base: FExpression): FExpression() +data class FMinusExpr(val base: FExpression): FExpression() +``` + +## Method definitions + +```kotlin +data class FMethodDeclaration( + override var name: String, + val params: MutableList, + // body is null for abstract methods. We want to distinguish + // between abstract methods and methods with an empty body + var body: FBody? = null) : FClassMember(), Named +``` + +## Partitions + +Partitions are special constructs needed for LionWeb interoperability to represent collections of ASTs. + +They are *not* represented as Kolasu nodes (the annotation *LionWebPartition* has been removed) + +## Placeholders + +Placeholders are used to create patterns: + +```kotlin +data class RPGPlaceholderExpr(@property:Internal override var placeholderName: String? = null) : + Expression(), PlaceholderElement { + @property:Internal + override val multiplePlaceholderElement: Boolean + get() = false +} + +data class RPGPlaceholderMultiStmt(@property:Internal override var placeholderName: String? = null) : + Statement(), PlaceholderElement { + @property:Internal + override val multiplePlaceholderElement: Boolean + get() = true +} + +``` + +Why do we need pattern? To match ASTs or produce following a certain structure. Consider this example: + +```kotlin +this.statementByExample( +""" C FOR I = `expr:s` TO `expr:e` + C `stmt*:b` + C ENDFOR""", +"""for (int I = `expr:s`;I <= `expr:e`;I++) { `stmt*:b` }""", +variants= listOf("I") +) +``` + +This piece of code: + +- Match in an RPG AST every for statement defining a variable called I, *irrespectively of what the starting and ending expressions are and irrespectively of the statements it contains* +- It then produce a piece of Java code, filling in the translation of the corresponding elements captured in the RPG code + +For example, if we parse: + +```kotlin + C FOR I = 1 TO 2 + C "hello" DSPLY + C "world" DSPLY + C ENDFOR +``` + +The output Java code would be: + +```java +for (int I = 1;I <= 2;I++) { + System.out.println("hello"); + System.out.println("world"); +} +``` + +In order to represent “AST with holes” we use the placeholder parsing rules and the placeholder AST elements. + +A demo of this feature is presented in this video: [Let's talk about transpilers](https://youtu.be/d-ulE5Trb0s?t=1754). \ No newline at end of file diff --git a/documentation/traversing.md b/Writerside/topics/ASTTraversalAndQuerying.md similarity index 86% rename from documentation/traversing.md rename to Writerside/topics/ASTTraversalAndQuerying.md index e4b7686..b886faa 100644 --- a/documentation/traversing.md +++ b/Writerside/topics/ASTTraversalAndQuerying.md @@ -1,53 +1,52 @@ -# Traversing the AST - -To process the information contained in an AST, we'll have to _traverse_ it. That is, process each node in a given sequence. - -## Manual Traversal - -We can traverse a tree "manually" i.e. programmatically following all relevant references in the nodes. E.g. start from -the root node `Program`, then visit the contents of a child collection `statements`, and so on. - -However, this approach produces a traversal algorithm that depends on the precise structure of the AST. Therefore, the -algorithm is not reusable, and this approach doesn't scale over a growing number of node instances and node types. -It's also brittle in evolving code bases where the structure of the AST changes over time. - -## Traversal Functions - -So, StarLasu provides a number of generic traversal functions that we can apply and adapt to our ASTs for our use cases. -In languages where it's possible to do so, StarLasu exposes these functions as "extension methods" on the `Node` class. -These functions do not return a collection of nodes, rather they are "generators" or "iterators" and similar concepts -depending on the implementation language. These are facilities that return or "yield" one node after the other, in the intended sequence. - -The basic traversal method is `walk`, that visits all the nodes in the tree in a depth-first sequence. - -_See an example in:_ -- [Kolasu](https://github.com/Strumenta/kolasu/blob/ca96cc9200e26cee5674c0809fdff557b0ccbbc9/core/src/test/kotlin/com/strumenta/kolasu/traversing/TraversingStructurallyTest.kt#L188-L196) -- [Pylasu](https://github.com/Strumenta/pylasu/blob/a57368920b0dd14868347690dea86066c1e17727/tests/test_traversing.py#L21-L23) -- [Tylasu](https://github.com/Strumenta/tylasu/blob/29c5dcac2384f03f4911b80d4f310a09a754b0ef/tests/traversing.test.ts#L45C12-L46) - -Most other methods are derived from `walk` and may change the order of traversal. - -An exception is methods that travel upwards from a node to its ancestors, rather than downwards to its children. -This is the case of the `walkAncestors` method. - -## Listeners and Visitors - -In StarLasu we do not encourage the usage of visitors or listeners, that are commonly used with ANTLR for example. -These have to be generated and produce interfaces with a number of methods depending on the number of node types, which can grow high. -Moreover, listeners/visitors make it difficult to organize code in a structured manner. - -## Finding Nodes of a Given Type - -Given an AST there is one essential operation we may want to do: find nodes of a given type. We can search for nodes of a given type in two -directions: looking among descendants of the current node and look among ancestors. - -For example, given a field declaration we may want to know in which class is declared. In that case we will look for the closest ancestor -of type ClassDeclaration. - -On the other hand, we may identify all the return statement inside a MethodDeclaration. In that case we will look for them among the descendants -of the node. - -_See in:_ -- [Kolasu](https://github.com/Strumenta/kolasu/tree/master/core/src/main/kotlin/com/strumenta/kolasu/traversing) -- [Tylasu](https://github.com/Strumenta/tylasu/tree/master/src/traversing) - +# AST Traversal and Querying + +To process the information contained in an AST, we'll have to _traverse_ it. That is, process each node in a given sequence. + +## Manual Traversal + +We can traverse a tree "manually" i.e. programmatically following all relevant references in the nodes. E.g. start from +the root node `Program`, then visit the contents of a child collection `statements`, and so on. + +However, this approach produces a traversal algorithm that depends on the precise structure of the AST. Therefore, the +algorithm is not reusable, and this approach doesn't scale over a growing number of node instances and node types. +It's also brittle in evolving code bases where the structure of the AST changes over time. + +## Traversal Functions + +So, StarLasu provides a number of generic traversal functions that we can apply and adapt to our ASTs for our use cases. +In languages where it's possible to do so, StarLasu exposes these functions as "extension methods" on the `Node` class. +These functions do not return a collection of nodes, rather they are "generators" or "iterators" and similar concepts +depending on the implementation language. These are facilities that return or "yield" one node after the other, in the intended sequence. + +The basic traversal method is `walk`, that visits all the nodes in the tree in a depth-first sequence. + +_See an example in:_ +- [Kolasu](https://github.com/Strumenta/kolasu/blob/ca96cc9200e26cee5674c0809fdff557b0ccbbc9/core/src/test/kotlin/com/strumenta/kolasu/traversing/TraversingStructurallyTest.kt#L188-L196) +- [Pylasu](https://github.com/Strumenta/pylasu/blob/a57368920b0dd14868347690dea86066c1e17727/tests/test_traversing.py#L21-L23) +- [Tylasu](https://github.com/Strumenta/tylasu/blob/29c5dcac2384f03f4911b80d4f310a09a754b0ef/tests/traversing.test.ts#L45C12-L46) + +Most other methods are derived from `walk` and may change the order of traversal. + +An exception is methods that travel upwards from a node to its ancestors, rather than downwards to its children. +This is the case of the `walkAncestors` method. + +## Listeners and Visitors + +In StarLasu we do not encourage the usage of visitors or listeners, that are commonly used with ANTLR for example. +These have to be generated and produce interfaces with a number of methods depending on the number of node types, which can grow high. +Moreover, listeners/visitors make it difficult to organize code in a structured manner. + +## Finding Nodes of a Given Type + +Given an AST there is one essential operation we may want to do: find nodes of a given type. We can search for nodes of a given type in two +directions: looking among descendants of the current node and look among ancestors. + +For example, given a field declaration we may want to know in which class is declared. In that case we will look for the closest ancestor +of type ClassDeclaration. + +On the other hand, we may identify all the return statement inside a MethodDeclaration. In that case we will look for them among the descendants +of the node. + +_See in:_ +- [Kolasu](https://github.com/Strumenta/kolasu/tree/master/core/src/main/kotlin/com/strumenta/kolasu/traversing) +- [Tylasu](https://github.com/Strumenta/tylasu/tree/master/src/traversing) \ No newline at end of file diff --git a/Writerside/topics/ChiselMethod.md b/Writerside/topics/ChiselMethod.md new file mode 100644 index 0000000..e7d9fac --- /dev/null +++ b/Writerside/topics/ChiselMethod.md @@ -0,0 +1,3 @@ +# The Chisel Method + +_To be written_ \ No newline at end of file diff --git a/Writerside/topics/CodeGeneration.md b/Writerside/topics/CodeGeneration.md new file mode 100644 index 0000000..66060ba --- /dev/null +++ b/Writerside/topics/CodeGeneration.md @@ -0,0 +1,3 @@ +# Code Generation + +_To be written_ \ No newline at end of file diff --git a/documentation/ast_common_elements.md b/Writerside/topics/CommonElements.md similarity index 91% rename from documentation/ast_common_elements.md rename to Writerside/topics/CommonElements.md index 80c6715..3ce2976 100644 --- a/documentation/ast_common_elements.md +++ b/Writerside/topics/CommonElements.md @@ -10,5 +10,4 @@ They are: _See in [Kolasu](https://github.com/Strumenta/kolasu/blob/master/core/src/main/kotlin/com/strumenta/kolasu/model/CommonElements.kt)_ -_This is not yet supported in other StarLasu libraries._ - +_This is not yet supported in other StarLasu libraries._ \ No newline at end of file diff --git a/documentation/parsers-cross-platform.md b/Writerside/topics/CrossPlatformParsers.md similarity index 96% rename from documentation/parsers-cross-platform.md rename to Writerside/topics/CrossPlatformParsers.md index 8a300ab..db0a0f9 100644 --- a/documentation/parsers-cross-platform.md +++ b/Writerside/topics/CrossPlatformParsers.md @@ -8,5 +8,4 @@ On the other hand we want to avoid having to rewrite the parsers for the same la - generate AST classes on the other platforms from the metamodel - generate an AST unserializers from the metamodel -In this way we could write a parser for RPG in Kotlin, using Kolasu. We would then automatically generate equivalent AST classes in Pylasu, and code to unserialize an AST instantiating those AST classes. In the end, we would obtain a parser usable from Python, which expose AST classes in Python. The implementation would call the parser written in Kotlin, obtain the AST serialized and unserialize it behind the scenes. - +In this way we could write a parser for RPG in Kotlin, using Kolasu. We would then automatically generate equivalent AST classes in Pylasu, and code to unserialize an AST instantiating those AST classes. In the end, we would obtain a parser usable from Python, which expose AST classes in Python. The implementation would call the parser written in Kotlin, obtain the AST serialized and unserialize it behind the scenes. \ No newline at end of file diff --git a/Writerside/topics/DocumentationGeneration.md b/Writerside/topics/DocumentationGeneration.md new file mode 100644 index 0000000..38620a4 --- /dev/null +++ b/Writerside/topics/DocumentationGeneration.md @@ -0,0 +1,3 @@ +# AST Documentation + +_To be written_ \ No newline at end of file diff --git a/Writerside/topics/DualCodeModelAPIs.md b/Writerside/topics/DualCodeModelAPIs.md new file mode 100644 index 0000000..a66314d --- /dev/null +++ b/Writerside/topics/DualCodeModelAPIs.md @@ -0,0 +1,3 @@ +# The Dual Code Model APIs + +_To be written_ \ No newline at end of file diff --git a/documentation/emf.md b/Writerside/topics/EMFInteroperability.md similarity index 89% rename from documentation/emf.md rename to Writerside/topics/EMFInteroperability.md index 17da004..0bd820d 100644 --- a/documentation/emf.md +++ b/Writerside/topics/EMFInteroperability.md @@ -1,21 +1,20 @@ -# EMF - -The Eclipse Modeling Framework (EMF) has been very successful in the MDE world. It can be regarded as an exchange format supported by different tools. - -For this reason we built support for exporting Metamodels and Models to EMF. We in particular support the serialization to EMF-JSON. EMF-JSON is not as -well-defined and supported as XMI (based on XML), but the request for JSON-based formats on some of the platforms supported brought us to focus on it. - -In our case, _metamodels_ are the definition of the node types. For example, they specify which properties each node has. - -Instead, we can represent and serialize ASTs as _models_ (i.e., instances of metamodels). A model describes the value of each node's properties, and the -relationships among the nodes. - -In some of the StarLasu libraries, we can also _import_ EMF metamodels and models (generated with other StarLasu libraries) -in order to _consume_ the results of a tool (e.g. a parser) from a different language. This is what the -[Strumenta Playground](https://playground.strumenta.com/) web app does, for example. This is also useful for [cross-platform parsers](parsers-cross-platform.md). - -Read more about this topic in: -- [Kolasu](https://javadoc.io/doc/com.strumenta.kolasu/kolasu-emf/latest/index.html) ([source code](https://github.com/Strumenta/kolasu/tree/master/emf)) -- [Pylasu](https://pylasu.readthedocs.io/en/latest/pylasu.emf.html) ([source code](https://github.com/Strumenta/pylasu/tree/master/pylasu/emf)) _note: support in Pylasu is a work in progress._ -- [Tylasu](https://strumenta.github.io/tylasu/modules/interop_ecore.html) ([source code](https://github.com/Strumenta/tylasu/blob/master/src/interop/ecore.ts)) - +# EMF Interoperability + +The Eclipse Modeling Framework (EMF) has been very successful in the MDE world. It can be regarded as an exchange format supported by different tools. + +For this reason we built support for exporting Metamodels and Models to EMF. We in particular support the serialization to EMF-JSON. EMF-JSON is not as +well-defined and supported as XMI (based on XML), but the request for JSON-based formats on some of the platforms supported brought us to focus on it. + +In our case, _metamodels_ are the definition of the node types. For example, they specify which properties each node has. + +Instead, we can represent and serialize ASTs as _models_ (i.e., instances of metamodels). A model describes the value of each node's properties, and the +relationships among the nodes. + +In some of the StarLasu libraries, we can also _import_ EMF metamodels and models (generated with other StarLasu libraries) +in order to _consume_ the results of a tool (e.g. a parser) from a different language. This is what the +[Strumenta Playground](https://playground.strumenta.com/) web app does, for example. This is also useful for [cross-platform parsers](parsers-cross-platform.md). + +Read more about this topic in: +- [Kolasu](https://javadoc.io/doc/com.strumenta.kolasu/kolasu-emf/latest/index.html) ([source code](https://github.com/Strumenta/kolasu/tree/master/emf)) +- [Pylasu](https://pylasu.readthedocs.io/en/latest/pylasu.emf.html) ([source code](https://github.com/Strumenta/pylasu/tree/master/pylasu/emf)) _note: support in Pylasu is a work in progress._ +- [Tylasu](https://strumenta.github.io/tylasu/modules/interop_ecore.html) ([source code](https://github.com/Strumenta/tylasu/blob/master/src/interop/ecore.ts)) diff --git a/Writerside/topics/EditorSupport.md b/Writerside/topics/EditorSupport.md new file mode 100644 index 0000000..10ab08e --- /dev/null +++ b/Writerside/topics/EditorSupport.md @@ -0,0 +1,3 @@ +# Editor Support + +_To be written_ \ No newline at end of file diff --git a/Writerside/topics/Interoperability.md b/Writerside/topics/Interoperability.md new file mode 100644 index 0000000..2d37a9d --- /dev/null +++ b/Writerside/topics/Interoperability.md @@ -0,0 +1,3 @@ +# Interoperability + +_To be written_ \ No newline at end of file diff --git a/documentation/naming.md b/Writerside/topics/Naming.md similarity index 90% rename from documentation/naming.md rename to Writerside/topics/Naming.md index 5d81a88..cb00a40 100644 --- a/documentation/naming.md +++ b/Writerside/topics/Naming.md @@ -7,5 +7,4 @@ Two interfaces are defined: For example, a FunctionDeclaration will be PossiblyNamed in languages which permits anonymous functions. -In the StarLasu libraries we provide special support for things which are Named or PossiblyNamed. We can used these interfaces in [Symbol Resolution](https://github.com/Strumenta/StarLasu/blob/main/documentation/symbol_resolution.md), for example. - +In the StarLasu libraries we provide special support for things which are Named or PossiblyNamed. We can used these interfaces in [Symbol Resolution](https://github.com/Strumenta/StarLasu/blob/main/documentation/symbol_resolution.md), for example. \ No newline at end of file diff --git a/Writerside/topics/OriginAndDestination.md b/Writerside/topics/OriginAndDestination.md new file mode 100644 index 0000000..0af0c13 --- /dev/null +++ b/Writerside/topics/OriginAndDestination.md @@ -0,0 +1,22 @@ +# Origin and Destination + +A Node can have an Origin. An Origin indicates where the nodes come from. + +Let's see some examples: + +In the case of Nodes built by a parser, the Origin will indicate a point in some source file. +In the case of Nodes built by a transformer, the Origin may indicate some Node which provided the original information. +Consider this example: we are building a transpiler from RPG to Java. + +We first build an RPG parser that given RPG code product an RPG AST. The root of the AST may have type +RPGCompilationUnit. The origin of an RPGCompilationUnit will represent the RPG file name and the position in that file. +In the case of the root a position ranging from the first character of the first line, to the last character of the last line. + +Then we build a transformer that given an RPG AST produces a Java AST. The root of such AST may have type +JavaCompilationUnit. The origin of a JavaCompilationUnit will refer to another node, in this case having type +RPGCompilationUnit. + +In a Node we may also indicate what can be created from a Node. For example, if we build programmatically +an AST and then we want to generate code from it, we may want to specify for each node what portion of code +has been generated. For example, a node representing an If Statement in Java could end up being represented by a 5 +lines of Java code inside a large Java file. In that case, the Destination will specify the name of the file and the lines and columns representing our If Statement. \ No newline at end of file diff --git a/documentation/parsetree_to_ast.md b/Writerside/topics/ParseTreeToASTMapping.md similarity index 98% rename from documentation/parsetree_to_ast.md rename to Writerside/topics/ParseTreeToASTMapping.md index 1821fe6..ef1f8c5 100644 --- a/documentation/parsetree_to_ast.md +++ b/Writerside/topics/ParseTreeToASTMapping.md @@ -34,5 +34,4 @@ val cu = CU( Read more about this topic for: - [Kolasu](https://javadoc.io/doc/com.strumenta.kolasu/kolasu-core/latest/com/strumenta/kolasu/mapping/ParseTreeToASTTransformer.html) ([source code](https://github.com/Strumenta/kolasu/tree/master/core/src/main/kotlin/com/strumenta/kolasu/mapping)) - [Pylasu](https://pylasu.readthedocs.io/en/latest/pylasu.mapping.html#pylasu-mapping-parse-tree-to-ast-transformer-module) -- [Tylasu](https://strumenta.github.io/tylasu/classes/mapping.parsetreetoasttransformer.html) - +- [Tylasu](https://strumenta.github.io/tylasu/classes/mapping.parsetreetoasttransformer.html) \ No newline at end of file diff --git a/Writerside/topics/Position.md b/Writerside/topics/Position.md new file mode 100644 index 0000000..541d655 --- /dev/null +++ b/Writerside/topics/Position.md @@ -0,0 +1,8 @@ +# Position + +Each Node can have an associated position. It indicates a in textual files + +A Position has a start and an end Point. Each position is relative to a certain Source, which is optional. + +A Point has a Line and a Column (both integer values). The first line of a file is Line 1. +The first Column of each line is 0. The starting point of file has therefore Line=1 and Column=0. diff --git a/Writerside/topics/SemanticEnrichment.md b/Writerside/topics/SemanticEnrichment.md new file mode 100644 index 0000000..7352b72 --- /dev/null +++ b/Writerside/topics/SemanticEnrichment.md @@ -0,0 +1,12 @@ +# Semantic Enrichment + +_To be written_ + +Symbol Resolution is the process of giving a meaning to the each name in the source +code. For example: + +* to connect a given use of a variable to its declaration; +* * to reconstruct the definition of a SQL column from an alias in a subquery; +* * to identify a user-defined therapy plan in a medical support DSL. + +Symbol resolution relies on the naming facilities in StarLasu. \ No newline at end of file diff --git a/documentation/serialization.md b/Writerside/topics/Serialization.md similarity index 83% rename from documentation/serialization.md rename to Writerside/topics/Serialization.md index a05f736..a6317f9 100644 --- a/documentation/serialization.md +++ b/Writerside/topics/Serialization.md @@ -4,5 +4,4 @@ In addition to what is described here, there is also EMF serialization, which is StarLasu supports exporting ASTs to JSON and XML. -_See in [Kolasu](https://github.com/Strumenta/kolasu/tree/master/core/src/main/kotlin/com/strumenta/kolasu/serialization)_. - +_See in [Kolasu](https://github.com/Strumenta/kolasu/tree/master/core/src/main/kotlin/com/strumenta/kolasu/serialization)_. \ No newline at end of file diff --git a/Writerside/topics/StarlasuOverview.md b/Writerside/topics/StarlasuOverview.md new file mode 100644 index 0000000..a604e30 --- /dev/null +++ b/Writerside/topics/StarlasuOverview.md @@ -0,0 +1,118 @@ + +# The Starlasu Approach + +The Starlasu approach provides a flexible, systematic, and extensible framework for creating tools like parsers, +transpilers, code analyzers, interpreters, code generators, and domain-specific languages (DSLs). + +The Starlasu approach was developed at [Strumenta](https://strumenta.com), drawing on a decade of experience in Language +Engineering projects. +It reflects lessons learned about what works and what doesn’t, +combining this expertise to deliver reliable and adaptable language processing systems. + +## What can you do with The Starlasu approach? + +The goal is to be able to define arbitrary Language Engineering applications, and flexibility is the main goal +of this approach. + +Concretely, the most common applications we write using these methods are: +1. **Domain-Specific Languages (DSLs)**: We define new tailored languages for specific domains with accompanying editors and execution engines (either interpreters or code generators). +2. **Transpilers**: Applications that translate code from one language to another. Typically from a legacy language such as RPG to a modern one such as Java. +3. **Parsers**: Tools that generate Code Models for code analysis, documentation generation, or as the initial step of +transpilers. We typically implements these Parsers that are then used to build code analysis tools, transpilers, or other systems. + +## Architectural Overview + +The Starlasu approach is centered around the idea of **Code Model**. +We have this component at the center and a constellation of other components interacting with it. + +![Overview of the Starlasu Approach](../images/StarlasuOverview.png) + +### The Code Model (or AST) +The **Code Model** represents a representation of the information present in the code, in a form that facilitates +reasoning about it and processing. It facilitates reasoning by being close to the mental model a developer would +have of the code and it facilitates processing through the [dualistic homogeneous and heterogeneous APIs](DualCodeModelAPIs.md). + +The Code Model is an evolution of the Abstract Syntax Tree (AST). + +### Surrounding Components +Around this central model, there are several essential components: +- **Parsers**: These components process code, and "understand it". This understanding is defined as the Code Model. +- **Code Generators**: These components produce code according to a Code Model. +- **Code Model Transformations**: These components produce a different Code Model, from an original Code Model. +- **Editors**: These components permit to edit code, either through textual, graphical, or projectional editors. +They provide Language Intelligence by using the underlying Code Model +- **Semantic Enrichment**: These components analyze a Code Model performing symbol resolution and type calculation over it, + and adding additional metadata to the original Code Model. +- **Interpreters**: They permit the execution of Code Models + +These components work in a star-like configuration around the AST, emphasizing modularity + +### Combining Components + +The components we just examined can be combined into different pipelines for different goals. +E.g., a parser will transform a lower-level parse tree into a higher-level AST, then perhaps resolve names +and references, and type-check the code. A transpiler will also perhaps transform the source code into some +intermediate representation, then into the target language AST, from which it will generate the resulting +code string. + +## What one gets from Starlasu? + +Starlasu consists of a mental framework and guidelines to build the different component. For example, we have a detailed +method to design and implement parsers (see [The Chisel Method](ChiselMethod.md)). + +One also gets a family of libraries, all built around the same principles but for different programming languages. In this +way one can conveniently use the Starlasu when programming in a multitude of programming languages. + +Starlasu is supported by a family of libraries, each supporting the application of The Starlasu approach on +different platforms: +- **Kolasu**: For the JVM (Java, Kotlin), expanding to Node.js and browser environments in version 1.6. +- **Tylasu**: For Node.js and browser environments. +- **Pylasu**: For Python. +- **Sharplasu**: For .NET. + +These libraries share a common architecture and are interoperable. This enable cross-platform development and +consistent tooling. + +### Feature Matrix + +Let's see more in detail what each library of the Starlasu family provides. +Below is a detailed matrix of the features provided by each of them: + +| Feature | Kolasu | Tylasu | Sharplasu | Pylasu | +|----------------------------------------------|----------|-----------------|-------------------|-----------------| +| [AST Representation](ASTRepresentation.md) | Stable | Stable | Stable | Stable | +| [Semantic Enrichment](SemanticEnrichment.md) | Complete | Not implemented | Complete | Not implemented | +| Code Generation | Stable | Not implemented | Under development | Not implemented | +| Parser Integration | Stable | Stable | Stable | Stable | +| Editor Support | Partial | Partial | Not implemented | Not implemented | +| Interoperability | Stable | Stable | Partial | Partial | +| AST Traversal and Querying | Stable | Stable | Stable | Stable | +| Transformation Framework | Stable | Stable | Stablee | Stable | +| AST Documentation | Stable | Stable | Stable | Stable | +| Testing | Stable | Stable | Stable | Stable | +| Validation | Stable | Stable | Stable | Stable | + +Where: +* **Stable** means that the functionality has been used in many projects and it is not expected to change +* **Complete** means that the functionality has been implemented. It has been validated but it could futher evolve +* **Partial** a portion of the features of the functionality have been implemented +* **Under development** the functionality is currently under development +* **Not implemented** the functionality is not implemented + +## Why the name Starlasu? + +Starlasu combines the concept of a star and the shortand for language support. Why a star? To indicate the fact that we +have at the center one element (the code model), and everything else operates on it either producing code models, +refining code models or consuming code model. + + +## Origin of The Starlasu approach +The Starlasu approach originated from Strumenta’s commitment to simplifying and standardizing language engineering. +In other words, we were not particularly keen reinventing the wheel and doing the same mistakes over and over, so we +wanted to capture what we learned in one-hundred or so language engineering projects to overcome the typical pitfalls, +get a design that was extensible, and that permitted to get where we want as fast and uneventfully as possible. + +To achieve that we distilled common patterns into reusable principles and libraries. These ideas have been influenced +by the background of each of us, and draw on experience from the Eclipse Modeling Framework community, +Model Driven Engineering, Projectional Editing and the work of giants in the community such as Markus Völter, Meinte Boersma, +Jos Warmer, Sascha Lißon and others. diff --git a/documentation/testing.md b/Writerside/topics/Testing.md similarity index 95% rename from documentation/testing.md rename to Writerside/topics/Testing.md index 19b295f..bfb5eaf 100644 --- a/documentation/testing.md +++ b/Writerside/topics/Testing.md @@ -1,25 +1,24 @@ -# Testing - -StarLasu offers support for comparing parse trees and ASTs. - -See `assertParseTreeStr`, `assertParsingResultsAreEqual`, and `assertASTsAreEqual` in Kolasu. - -## Coverage of the grammar - -Related to this, there is experimental support for verifying the Coverage of a grammar by the examples we have. See [CoverageListener](https://github.com/Strumenta/kolasu/blob/master/core/src/main/kotlin/com/strumenta/kolasu/parsing/coverage/CoverageListener.kt) in Kolasu. - -The goal is that, given a grammar and a set of examples, we want to understand: - -- How many possible paths in the grammar are covered -- Which alternatives are not covered, so that we can look for appropriate examples - -Another solution is to verify the coverage of the generated ANTLR Parser. - -## Performance testing - -_To be written._ - -## Test the parser on examples - -In practice it is often convenient to run the parser on a larget set of examples and just check if the parser can handle them without finding errors. - +# Testing + +StarLasu offers support for comparing parse trees and ASTs. + +See `assertParseTreeStr`, `assertParsingResultsAreEqual`, and `assertASTsAreEqual` in Kolasu. + +## Coverage of the grammar + +Related to this, there is experimental support for verifying the Coverage of a grammar by the examples we have. See [CoverageListener](https://github.com/Strumenta/kolasu/blob/master/core/src/main/kotlin/com/strumenta/kolasu/parsing/coverage/CoverageListener.kt) in Kolasu. + +The goal is that, given a grammar and a set of examples, we want to understand: + +- How many possible paths in the grammar are covered +- Which alternatives are not covered, so that we can look for appropriate examples + +Another solution is to verify the coverage of the generated ANTLR Parser. + +## Performance testing + +_To be written._ + +## Test the parser on examples + +In practice it is often convenient to run the parser on a larget set of examples and just check if the parser can handle them without finding errors. diff --git a/documentation/transformations.md b/Writerside/topics/TransformationFramework.md similarity index 97% rename from documentation/transformations.md rename to Writerside/topics/TransformationFramework.md index bb6c3e1..599d3ca 100644 --- a/documentation/transformations.md +++ b/Writerside/topics/TransformationFramework.md @@ -1,94 +1,93 @@ -# Transforming ASTs - -In general we may want to process ASTs and use their information to produce something else. For example, to generate a diagram or to generate code. - -A particular case is the transformation of an AST into another AST. This is typically done within transpilers. - -## Example of refactoring within the same language - -One usage of transformations is to perform refactoring within the same AST. For example, let's suppose that we have this language: - -``` -enum class Operator { - PLUS, MULT -} -sealed class Expression : Node() -data class IntLiteral(val value: Int) : Expression() -data class GenericBinaryExpression(val operator: Operator, val left: Expression, val right: Expression) : Node() -data class Mult(val left: Expression, val right: Expression) : Node() -data class Sum(val left: Expression, val right: Expression) : Node() -``` - -We then decide to transform an AST by removing instances of `GenericBinaryExpression` and replace them with `Mult` or `Sum`. We can do that in this way: - -``` -val myTransformer = ASTTransformer(allowGenericNode = false).apply { - registerNodeFactory(GenericBinaryExpression::class) { source: GenericBinaryExpression -> - when (source.operator) { - Operator.MULT -> Mult(transform(source.left) as Expression, transform(source.right) as Expression) - Operator.PLUS -> Sum(transform(source.left) as Expression, transform(source.right) as Expression) - } - } - // This may benefit of specific support: for example a NodeFactory that returns the same element - registerNodeFactory(IntLiteral::class) { source: IntLiteral -> source } -} -assertASTsAreEqual( - Mult(IntLiteral(7), IntLiteral(8)), - myTransformer.transform(GenericBinaryExpression(Operator.MULT, IntLiteral(7), IntLiteral(8)))!! -) -assertASTsAreEqual( - Sum(IntLiteral(7), IntLiteral(8)), - myTransformer.transform(GenericBinaryExpression(Operator.PLUS, IntLiteral(7), IntLiteral(8)))!! -) -``` - -## Example of translation to another language - -Let's consider two languages. In this example they have exactly the same structure: - -``` -sealed class ALangExpression : Node() -data class ALangIntLiteral(val value: Int) : ALangExpression() -data class ALangSum(val left: ALangExpression, val right: ALangExpression) : ALangExpression() -data class ALangMult(val left: ALangExpression, val right: ALangExpression) : ALangExpression() - -sealed class BLangExpression : Node() -data class BLangIntLiteral(val value: Int) : BLangExpression() -data class BLangSum(val left: BLangExpression, val right: BLangExpression) : BLangExpression() -data class BLangMult(val left: BLangExpression, val right: BLangExpression) : BLangExpression() -``` - -While this is a toy example it is true that many languages shares similar structures. Think of literals, mathematical operations, or basic control flow structures such as if-statements: they have the same structures in languages which are very different. - -We could build a transformer that given an AST of `ALang` produces the corresponding AST of `BLang`: - -``` -val myTransformer = ASTTransformer(allowGenericNode = false).apply { - registerNodeFactory(ALangIntLiteral::class) { source: ALangIntLiteral -> BLangIntLiteral(source.value) } - registerNodeFactory(ALangSum::class) { source: ALangSum -> - BLangSum(transform(source.left) as BLangExpression, transform(source.right) as BLangExpression) - } - registerNodeFactory(ALangMult::class) { source: ALangMult -> - BLangMult(transform(source.left) as BLangExpression, transform(source.right) as BLangExpression) - } -} -assertASTsAreEqual( - BLangMult( - BLangSum( - BLangIntLiteral(1), - BLangMult(BLangIntLiteral(2), BLangIntLiteral(3)) - ), - BLangIntLiteral(4) - ), - myTransformer.transform( - ALangMult( - ALangSum( - ALangIntLiteral(1), - ALangMult(ALangIntLiteral(2), ALangIntLiteral(3)) - ), - ALangIntLiteral(4) - ) - )!! -) -``` - +# Transformation Framework + +In general we may want to process ASTs and use their information to produce something else. For example, to generate a diagram or to generate code. + +A particular case is the transformation of an AST into another AST. This is typically done within transpilers. + +## Example of refactoring within the same language + +One usage of transformations is to perform refactoring within the same AST. For example, let's suppose that we have this language: + +``` +enum class Operator { + PLUS, MULT +} +sealed class Expression : Node() +data class IntLiteral(val value: Int) : Expression() +data class GenericBinaryExpression(val operator: Operator, val left: Expression, val right: Expression) : Node() +data class Mult(val left: Expression, val right: Expression) : Node() +data class Sum(val left: Expression, val right: Expression) : Node() +``` + +We then decide to transform an AST by removing instances of `GenericBinaryExpression` and replace them with `Mult` or `Sum`. We can do that in this way: + +``` +val myTransformer = ASTTransformer(allowGenericNode = false).apply { + registerNodeFactory(GenericBinaryExpression::class) { source: GenericBinaryExpression -> + when (source.operator) { + Operator.MULT -> Mult(transform(source.left) as Expression, transform(source.right) as Expression) + Operator.PLUS -> Sum(transform(source.left) as Expression, transform(source.right) as Expression) + } + } + // This may benefit of specific support: for example a NodeFactory that returns the same element + registerNodeFactory(IntLiteral::class) { source: IntLiteral -> source } +} +assertASTsAreEqual( + Mult(IntLiteral(7), IntLiteral(8)), + myTransformer.transform(GenericBinaryExpression(Operator.MULT, IntLiteral(7), IntLiteral(8)))!! +) +assertASTsAreEqual( + Sum(IntLiteral(7), IntLiteral(8)), + myTransformer.transform(GenericBinaryExpression(Operator.PLUS, IntLiteral(7), IntLiteral(8)))!! +) +``` + +## Example of translation to another language + +Let's consider two languages. In this example they have exactly the same structure: + +``` +sealed class ALangExpression : Node() +data class ALangIntLiteral(val value: Int) : ALangExpression() +data class ALangSum(val left: ALangExpression, val right: ALangExpression) : ALangExpression() +data class ALangMult(val left: ALangExpression, val right: ALangExpression) : ALangExpression() + +sealed class BLangExpression : Node() +data class BLangIntLiteral(val value: Int) : BLangExpression() +data class BLangSum(val left: BLangExpression, val right: BLangExpression) : BLangExpression() +data class BLangMult(val left: BLangExpression, val right: BLangExpression) : BLangExpression() +``` + +While this is a toy example it is true that many languages shares similar structures. Think of literals, mathematical operations, or basic control flow structures such as if-statements: they have the same structures in languages which are very different. + +We could build a transformer that given an AST of `ALang` produces the corresponding AST of `BLang`: + +``` +val myTransformer = ASTTransformer(allowGenericNode = false).apply { + registerNodeFactory(ALangIntLiteral::class) { source: ALangIntLiteral -> BLangIntLiteral(source.value) } + registerNodeFactory(ALangSum::class) { source: ALangSum -> + BLangSum(transform(source.left) as BLangExpression, transform(source.right) as BLangExpression) + } + registerNodeFactory(ALangMult::class) { source: ALangMult -> + BLangMult(transform(source.left) as BLangExpression, transform(source.right) as BLangExpression) + } +} +assertASTsAreEqual( + BLangMult( + BLangSum( + BLangIntLiteral(1), + BLangMult(BLangIntLiteral(2), BLangIntLiteral(3)) + ), + BLangIntLiteral(4) + ), + myTransformer.transform( + ALangMult( + ALangSum( + ALangIntLiteral(1), + ALangMult(ALangIntLiteral(2), ALangIntLiteral(3)) + ), + ALangIntLiteral(4) + ) + )!! +) +``` \ No newline at end of file diff --git a/Writerside/topics/Use-Cases.md b/Writerside/topics/Use-Cases.md new file mode 100644 index 0000000..e879adf --- /dev/null +++ b/Writerside/topics/Use-Cases.md @@ -0,0 +1,3 @@ +# Use Cases + +Here we group a few different Use Cases for the Starlasu Approach. \ No newline at end of file diff --git a/documentation/validation.md b/Writerside/topics/Validation.md similarity index 75% rename from documentation/validation.md rename to Writerside/topics/Validation.md index d1bf788..b9aaca1 100644 --- a/documentation/validation.md +++ b/Writerside/topics/Validation.md @@ -1,11 +1,10 @@ -# Validation - -We have a concept of `Issue`. Each issue has a message, a position, a IssueSeverity, and an IssueType. - -The `IssueType` can be lexical, syntactic, semantic, or translation. -The `IssueSeverity` can be info, warning, or error. - -Results are objects that provide a value and a collection of issues. - -_See in [Kolasu](https://github.com/Strumenta/kolasu/tree/master/core/src/main/kotlin/com/strumenta/kolasu/validation)_. - +# Validation + +We have a concept of `Issue`. Each issue has a message, a position, a IssueSeverity, and an IssueType. + +The `IssueType` can be lexical, syntactic, semantic, or translation. +The `IssueSeverity` can be info, warning, or error. + +Results are objects that provide a value and a collection of issues. + +_See in [Kolasu](https://github.com/Strumenta/kolasu/tree/master/core/src/main/kotlin/com/strumenta/kolasu/validation)_. \ No newline at end of file diff --git a/documentation/usecases/building-codegenerator.md b/Writerside/topics/usecases/building-codegenerator.md similarity index 98% rename from documentation/usecases/building-codegenerator.md rename to Writerside/topics/usecases/building-codegenerator.md index 469c042..66e8b5c 100644 --- a/documentation/usecases/building-codegenerator.md +++ b/Writerside/topics/usecases/building-codegenerator.md @@ -1,12 +1,12 @@ -# Building a code generator - -A code generator is a component that given an AST permits to generate code (i.e., text). - -It can be a component inside a [transpiler](building-transpiler.md), or it can be used by itself to output source code -from an AST that we've constructed programmatically (that is, explicitly creating and combining nodes, rather than -starting from the output of a parser). - -This use case is currently being improved in StarLasu. At the moment, we're experimenting in internal projects such as -StarLasu-Tools (to generate Kotlin and Python). In the future more support is planned to be added in StarLasu itself. - -For more information about code generation options and techniques, please refer to our [Guide to Code Generation](https://tomassetti.me/code-generation/). +# Building a code generator + +A code generator is a component that given an AST permits to generate code (i.e., text). + +It can be a component inside a [transpiler](building-transpiler.md), or it can be used by itself to output source code +from an AST that we've constructed programmatically (that is, explicitly creating and combining nodes, rather than +starting from the output of a parser). + +This use case is currently being improved in StarLasu. At the moment, we're experimenting in internal projects such as +StarLasu-Tools (to generate Kotlin and Python). In the future more support is planned to be added in StarLasu itself. + +For more information about code generation options and techniques, please refer to our [Guide to Code Generation](https://tomassetti.me/code-generation/). diff --git a/documentation/usecases/building-parser.md b/Writerside/topics/usecases/building-parser.md similarity index 98% rename from documentation/usecases/building-parser.md rename to Writerside/topics/usecases/building-parser.md index 959292c..f674d8d 100644 --- a/documentation/usecases/building-parser.md +++ b/Writerside/topics/usecases/building-parser.md @@ -1,26 +1,25 @@ -# Building a parser - -Building parsers is the most traditional use case. Kolasu was initially created with this goal in mind and tens of commercial parsers have been -implemented with it. - -Our preferred approach is to initially use an ANTLR parser and then wrap it with StarLasu ASTs. - -## Process - -Initially, we define an ANTLR grammar. Then, we define AST classes are using StarLasu. -An initial version may also be generated from the ANTLR grammar, by using StarLasu-Tools. - -We then organize the parser into a pipeline: - -1. First-stage parsing using ANTLR. We obtain a parse tree and, possibly, a series of errors. -2. Second-stage parsing. [The parse tree is mapped into the AST](../parsetree_to_ast.md) -3. Potentially, for some parsers, we perform additional steps, such as: - 1. [Symbol resolution](../symbol_resolution.md) - 2. [Type checking](../typesystem.md) - 3. Advanced calculations such as lineage, data flow analysis, linting, etc. - -The StarLasu ASTs provide a more convenient API with respect to the ANTLR APIs for parse trees. In fact, StarLasu comes -with additional features such as serialization and support for symbol resolution. Finally, when mapping the parse tree -into an AST, we drop implementation details of the grammar from the tree, and we organize the information in the most -convenient way for users. - +# Building a parser + +Building parsers is the most traditional use case. Kolasu was initially created with this goal in mind and tens of commercial parsers have been +implemented with it. + +Our preferred approach is to initially use an ANTLR parser and then wrap it with StarLasu ASTs. + +## Process + +Initially, we define an ANTLR grammar. Then, we define AST classes are using StarLasu. +An initial version may also be generated from the ANTLR grammar, by using StarLasu-Tools. + +We then organize the parser into a pipeline: +1. First-stage parsing using ANTLR. We obtain a parse tree and, possibly, a series of errors. +2. Second-stage parsing. [The parse tree is mapped into the AST](../parsetree_to_ast.md) +3. Potentially, for some parsers, we perform additional steps, such as: + 1. [Symbol resolution](../symbol_resolution.md) + 2. [Type checking](../typesystem.md) + 3. Advanced calculations such as lineage, data flow analysis, linting, etc. + +The StarLasu ASTs provide a more convenient API with respect to the ANTLR APIs for parse trees. In fact, StarLasu comes +with additional features such as serialization and support for symbol resolution. Finally, when mapping the parse tree +into an AST, we drop implementation details of the grammar from the tree, and we organize the information in the most +convenient way for users. + diff --git a/documentation/usecases/building-transpiler.md b/Writerside/topics/usecases/building-transpiler.md similarity index 98% rename from documentation/usecases/building-transpiler.md rename to Writerside/topics/usecases/building-transpiler.md index e3f193f..2743e55 100644 --- a/documentation/usecases/building-transpiler.md +++ b/Writerside/topics/usecases/building-transpiler.md @@ -1,19 +1,19 @@ -# Building a transpiler - -The general approach to design transpilers has been described in the article [How to write a transpiler](https://tomassetti.me/how-to-write-a-transpiler/). - -In general these are the steps: - -- The original code is parsed, obtaining the original AST (see [Building parsers](building-parser.md)) -- Transformations are performed to go from the original AST to the target AST (see [AST Transformations](../transformations.md)). If necessary, we can add intermediate transformation steps. -- Target code is generated from the target AST (see [Building code generators](building-codegenerator.md)) - -For example, suppose we want to translate RPG into Java: - -- We will use an [RPG Parser](https://strumenta.com/parser-for-rpg/), obtaining an RPG AST. -- We will implement AST transformations to transform the RPG AST into a Java AST, defined using e.g. Kolasu. The Java - AST specifically is a component that Strumenta has developed and can license. However, it's also possible to build - another one independently, using Kolasu or other StarLasu libraries. -- We will use a [code generator](building-codegenerator.md) to generate Java code from the Java AST. The code generator - could be a pre-built component, too, or we may develop it with support from StarLasu and possibly other libraries (e.g. a templating engine). - +# Building a transpiler + +The general approach to design transpilers has been described in the article [How to write a transpiler](https://tomassetti.me/how-to-write-a-transpiler/). + +In general these are the steps: + +- The original code is parsed, obtaining the original AST (see [Building parsers](building-parser.md)) +- Transformations are performed to go from the original AST to the target AST (see [AST Transformations](../transformations.md)). If necessary, we can add intermediate transformation steps. +- Target code is generated from the target AST (see [Building code generators](building-codegenerator.md)) + +For example, suppose we want to translate RPG into Java: + +- We will use an [RPG Parser](https://strumenta.com/parser-for-rpg/), obtaining an RPG AST. +- We will implement AST transformations to transform the RPG AST into a Java AST, defined using e.g. Kolasu. The Java + AST specifically is a component that Strumenta has developed and can license. However, it's also possible to build + another one independently, using Kolasu or other StarLasu libraries. +- We will use a [code generator](building-codegenerator.md) to generate Java code from the Java AST. The code generator + could be a pre-built component, too, or we may develop it with support from StarLasu and possibly other libraries (e.g. a templating engine). + diff --git a/Writerside/v.list b/Writerside/v.list new file mode 100644 index 0000000..2d12cb3 --- /dev/null +++ b/Writerside/v.list @@ -0,0 +1,5 @@ + + + + + diff --git a/Writerside/writerside.cfg b/Writerside/writerside.cfg new file mode 100644 index 0000000..d62d75d --- /dev/null +++ b/Writerside/writerside.cfg @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/configuration/listings-setup.tex b/configuration/listings-setup.tex deleted file mode 100644 index 99ca881..0000000 --- a/configuration/listings-setup.tex +++ /dev/null @@ -1,43 +0,0 @@ -\usepackage{xcolor} - -\usepackage{listings} - -\lstdefinelanguage{Kotlin}{ - comment=[l]{//}, - commentstyle={\color{gray}\ttfamily}, - emph={filter, first, firstOrNull, forEach, lazy, map, mapNotNull, println}, - emphstyle={\color{OrangeRed}}, - identifierstyle=\color{black}, - keywords={!in, !is, abstract, actual, annotation, as, as?, break, by, catch, class, companion, const, constructor, continue, crossinline, data, delegate, do, dynamic, else, enum, expect, external, false, field, file, final, finally, for, fun, get, if, import, in, infix, init, inline, inner, interface, internal, is, lateinit, noinline, null, object, open, operator, out, override, package, param, private, property, protected, public, receiveris, reified, return, return@, sealed, set, setparam, super, suspend, tailrec, this, throw, true, try, typealias, typeof, val, var, vararg, when, where, while}, - keywordstyle={\color{NavyBlue}\bfseries}, - morecomment=[s]{/*}{*/}, - morestring=[b]", - morestring=[s]{"""*}{*"""}, - ndkeywords={@Deprecated, @JvmField, @JvmName, @JvmOverloads, @JvmStatic, @JvmSynthetic, Array, Byte, Double, Float, Int, Integer, Iterable, Long, Runnable, Short, String, Any, Unit, Nothing}, - ndkeywordstyle={\color{BurntOrange}\bfseries}, - sensitive=true, - stringstyle={\color{ForestGreen}\ttfamily}, -} - -\lstset{ - basicstyle=\ttfamily, - numbers=left, - keywordstyle=\color[rgb]{0.13,0.29,0.53}\bfseries, - stringstyle=\color[rgb]{0.31,0.60,0.02}, - commentstyle=\color[rgb]{0.56,0.35,0.01}\itshape, - numberstyle=\footnotesize, - stepnumber=1, - numbersep=5pt, - backgroundcolor=\color[RGB]{248,248,248}, - showspaces=false, - showstringspaces=false, - showtabs=false, - tabsize=2, - captionpos=b, - breaklines=true, - breakatwhitespace=true, - breakautoindent=true, - escapeinside={\%*}{*)}, - linewidth=\textwidth, - basewidth=0.5em, -} \ No newline at end of file diff --git a/css/documentation.css b/css/documentation.css deleted file mode 100644 index a9e3f84..0000000 --- a/css/documentation.css +++ /dev/null @@ -1,328 +0,0 @@ -/* - * I add this to html files generated with pandoc. - */ - -html { - font-size: 100%; - overflow-y: scroll; - -webkit-text-size-adjust: 100%; - -ms-text-size-adjust: 100%; -} - -body { - color: #444; - font-family: Georgia, Palatino, 'Palatino Linotype', Times, 'Times New Roman', serif; - font-size: 12px; - line-height: 1.7; - padding: 1em; - margin: auto; - max-width: 42em; - background: #fefefe; -} - -a { - color: #0645ad; - text-decoration: none; -} - -a:visited { - color: #0b0080; -} - -a:hover { - color: #06e; -} - -a:active { - color: #faa700; -} - -a:focus { - outline: thin dotted; -} - -*::-moz-selection { - background: rgba(255, 255, 0, 0.3); - color: #000; -} - -*::selection { - background: rgba(255, 255, 0, 0.3); - color: #000; -} - -a::-moz-selection { - background: rgba(255, 255, 0, 0.3); - color: #0645ad; -} - -a::selection { - background: rgba(255, 255, 0, 0.3); - color: #0645ad; -} - -p { - margin: 1em 0; -} - -img { - max-width: 100%; -} - -h1, h2, h3, h4, h5, h6 { - color: #111; - line-height: 125%; - margin-top: 2em; - font-weight: normal; -} - -h4, h5, h6 { - font-weight: bold; -} - -h1 { - font-size: 2.5em; -} - -h2 { - font-size: 2em; -} - -h3 { - font-size: 1.5em; -} - -h4 { - font-size: 1.2em; -} - -h5 { - font-size: 1em; -} - -h6 { - font-size: 0.9em; -} - -blockquote { - color: #666666; - margin: 0; - padding-left: 3em; - border-left: 0.5em #EEE solid; -} - -hr { - display: block; - height: 2px; - border: 0; - border-top: 1px solid #aaa; - border-bottom: 1px solid #eee; - margin: 1em 0; - padding: 0; -} - -pre, code, kbd, samp { - color: #000; - font-family: monospace, monospace; - _font-family: 'courier new', monospace; - font-size: 0.98em; -} - -pre { - white-space: pre; - white-space: pre-wrap; - word-wrap: break-word; -} - -b, strong { - font-weight: bold; -} - -dfn { - font-style: italic; -} - -ins { - background: #ff9; - color: #000; - text-decoration: none; -} - -mark { - background: #ff0; - color: #000; - font-style: italic; - font-weight: bold; -} - -sub, sup { - font-size: 75%; - line-height: 0; - position: relative; - vertical-align: baseline; -} - -sup { - top: -0.5em; -} - -sub { - bottom: -0.25em; -} - -ul, ol { - margin: 1em 0; - padding: 0 0 0 2em; -} - -li p:last-child { - margin-bottom: 0; -} - -ul ul, ol ol { - margin: .3em 0; -} - -dl { - margin-bottom: 1em; -} - -dt { - font-weight: bold; - margin-bottom: .8em; -} - -dd { - margin: 0 0 .8em 2em; -} - -dd:last-child { - margin-bottom: 0; -} - -img { - border: 0; - -ms-interpolation-mode: bicubic; - vertical-align: middle; -} - -figure { - display: block; - text-align: center; - margin: 1em 0; -} - -figure img { - border: none; - margin: 0 auto; -} - -figcaption { - font-size: 0.8em; - font-style: italic; - margin: 0 0 .8em; -} - -table { - margin-bottom: 2em; - border-bottom: 1px solid #ddd; - border-right: 1px solid #ddd; - border-spacing: 0; - border-collapse: collapse; -} - -table th { - padding: .2em 1em; - background-color: #eee; - border-top: 1px solid #ddd; - border-left: 1px solid #ddd; -} - -table td { - padding: .2em 1em; - border-top: 1px solid #ddd; - border-left: 1px solid #ddd; - vertical-align: top; -} - -.author { - font-size: 1.2em; - text-align: center; -} - -@media only screen and (min-width: 480px) { - body { - font-size: 14px; - } -} -@media only screen and (min-width: 768px) { - body { - font-size: 16px; - } -} -@media print { - * { - background: transparent !important; - color: black !important; - filter: none !important; - -ms-filter: none !important; - } - - body { - font-size: 12pt; - max-width: 100%; - } - - a, a:visited { - text-decoration: underline; - } - - hr { - height: 1px; - border: 0; - border-bottom: 1px solid black; - } - - a[href]:after { - content: " (" attr(href) ")"; - } - - abbr[title]:after { - content: " (" attr(title) ")"; - } - - .ir a:after, a[href^="javascript:"]:after, a[href^="#"]:after { - content: ""; - } - - pre, blockquote { - border: 1px solid #999; - padding-right: 1em; - page-break-inside: avoid; - } - - tr, img { - page-break-inside: avoid; - } - - img { - max-width: 100% !important; - } - - @page :left { - margin: 15mm 20mm 15mm 10mm; -} - - @page :right { - margin: 15mm 10mm 15mm 20mm; -} - - p, h2, h3 { - orphans: 3; - widows: 3; - } - - h2, h3 { - page-break-after: avoid; - } -} \ No newline at end of file diff --git a/documentation/README.md b/documentation/README.md deleted file mode 100644 index d023387..0000000 --- a/documentation/README.md +++ /dev/null @@ -1,111 +0,0 @@ -# StarLasu: Overview - -StarLasu is a methodology to design and implement Language Engineering applications. -The methodology is supported by a set of libraries that permit its application on different platforms. - -The libraries are: - -- [Kolasu](https://github.com/strumenta/kolasu), for the implementation on the JVM (and in particular with Kotlin and Java) -- [Tylasu](https://github.com/strumenta/tylasu), for the implementation on Node.js and on the browser, using Typescript or Javascript -- [Pylasu](https://github.com/strumenta/pylasu), for the implementation with Python -- [Sharplasu](https://github.com/strumenta/sharplasu), for the implementation with C# - -Examples of applications that could use StarLasu: - -- Transpilers -- Parsers -- Code editors -- Code analyzers -- Linters -- Refactoring tools - -StarLasu libraries permit to work with Abstract Syntax Trees (ASTs). These are tree-like data structures to represent -the information contained in a piece of formal language or "code". For example, the statements in a procedural program, -the data elements in a SQL query, or the steps in a business workflow. - -We can use the StarLasu libraries to build different modules producing ASTs, enriching ASTs, transforming ASTs, and -finally consuming ASTs. These modules can be combined into different _pipelines_ for different goals. E.g., a parser -will transform a lower-level parse tree into a higher-level AST, then perhaps resolve names and references, and -type-check the code. A transpiler will also perhaps transform the source code into some intermediate representation, -then into the target language AST, from which it will generate the resulting code string. - -The name StarLasu is derived from `*lasu`, to indicate all the libraries part of the family. The suffix `lasu` stands -for Language Support. The original library has been Kolasu, which stands for _Kotlin Language Support_. Successive -libraries have taken derived names. - -## Support on the different platforms - -The different libraries have a different level of maturity. While eventually all libraries should be aligned, as of this -moment Kolasu has all the features, Tylasu has most of them, while Pylasu has support for the main features. - -## Goal of this document - -The goal of this document is to list the features provided by StarLasu and to describe the strategy suggested for -implementing certain features. It should also help ensuring that the different libraries for the different languages -provide the same features with the same logic. - -This is not a substitution for library-specific tutorials. Please refer to each individual library for additional -resources, API documentation, etc. - -## Typical usages of StarLasu - -These are the most typical applications of StarLasu: - -- [Building a parser](usecases/building-parser.md) -- [Building a code generator](usecases/building-codegenerator.md) -- [Building a transpiler](usecases/building-transpiler.md) - -StarLasu can be used to develop other Language Engineering applications (for example, inside editors, to provide -Language Intelligence). One of the goals of StarLasu is indeed to provide modularity, in according to the principles of -Model-Driven Development. Because of this, the different modules built with StarLasu can be combined in original ways, -for many different purposes. - -## Features of StarLasu - -At its core StarLasu permits to [define ASTs](https://github.com/Strumenta/StarLasu/blob/main/documentation/ast_definition.md). - -The nodes of each AST can specify [positions](https://github.com/Strumenta/StarLasu/blob/main/documentation/position.md). Related to this topic, nodes can specify their origin and destination (i.e., the object they've been generated from, and the object that's been generated from them). -This is discussed under [Origin and Destination](https://github.com/Strumenta/StarLasu/blob/main/documentation/origin_and_destination.md). - -StarLasu provides APIs for [traversing ASTs](https://github.com/Strumenta/StarLasu/blob/main/documentation/traversing.md) and for [transforming ASTs](https://github.com/Strumenta/StarLasu/blob/main/documentation/transformations.md). - -On top of the above core features, StarLasu builds several advanced concepts, detailed in the following sections. - -### Symbol Resolution - -[Symbol Resolution](https://github.com/Strumenta/StarLasu/blob/main/documentation/symbol_resolution.md) -is the process of giving a _meaning_ to the each _name_ in the source code. For example: - -* to connect a given use of a variable to its declaration; -* to reconstruct the definition of a SQL column from an alias in a subquery; -* to identify a user-defined therapy plan in a medical support DSL. - -Symbol resolution relies on the [naming](https://github.com/Strumenta/StarLasu/blob/main/documentation/naming.md) -facilities in StarLasu. - -### Type Systems - -We're also planning to add support for the definition of [type systems](https://github.com/Strumenta/StarLasu/blob/main/documentation/typesystem.md). - -### Code Generation - -Initially, with StarLasu we focused on creating parsers. However, ASTs are also useful to build code generators. -For this reason, work is going on to support [code generation](https://github.com/Strumenta/StarLasu/blob/main/documentation/code_generation.md) -in StarLasu. When we have both a parser and a code generator for the same language, we can define a -[language module](https://github.com/Strumenta/StarLasu/blob/main/documentation/language_module.md). - -### Other Features - -Other APIs include: - -- [Printing ASTs for debugging purposes](https://github.com/Strumenta/StarLasu/blob/main/documentation/debug_print_format.md) -- [Serialization of ASTs to different formats](https://github.com/Strumenta/StarLasu/blob/main/documentation/serialization.md) -- [Definition of common elements across languages](https://github.com/Strumenta/StarLasu/blob/main/documentation/ast_common_elements.md) -- [Definition of Command Line Tools](https://github.com/Strumenta/StarLasu/blob/main/documentation/cli_tools.md) -- [Translation of ANTLR parse trees into ASTs](https://github.com/Strumenta/StarLasu/blob/main/documentation/parsetree_to_ast.md) -- [Validation](https://github.com/Strumenta/StarLasu/blob/main/documentation/validation.md) -- [Testing](https://github.com/Strumenta/StarLasu/blob/main/documentation/testing.md) -- [EMF](https://github.com/Strumenta/StarLasu/blob/main/documentation/emf.md) -- [Strumenta Playground](https://github.com/Strumenta/StarLasu/blob/main/documentation/playground.md) -- [Build cross-platform Parsers](https://github.com/Strumenta/StarLasu/blob/main/documentation/parsers-cross-platform.md) - diff --git a/documentation/ast_definition.md b/documentation/ast_definition.md deleted file mode 100644 index 43c5b68..0000000 --- a/documentation/ast_definition.md +++ /dev/null @@ -1,122 +0,0 @@ -# AST Definition - -An Abstract Syntax Tree (AST) defined with StarLasu, has these characteristics: - -* It has a single root -* Each node has a type -* Each node has a parent, with the exception of root node -* Each type of node has: - - a package it belongs to - - a name - - a list of properties -* Each property of a node may represent an attribute, a containment relation, or a reference relation. -* Each property has a name and a multiplicity. Multiplicity can be 1, 0..1, 0..* . Every kind of property can have all these kinds of multiplicity (i.e., we can have references with multiple values) -* Each node should be part of exactly one containment relation, except for the root node which should not be contained by any other node -* A Reference Relation may or may not have an associated name. - -All StarLasu implementations target object-oriented languages. So, the preferred way of defining a node type is -__by inheritance__, i.e. subclassing the `Node` class. -[[Kolasu]](https://javadoc.io/doc/com.strumenta.kolasu/kolasu-core/latest/com/strumenta/kolasu/model/Node.html) -[[Pylasu]](https://pylasu.readthedocs.io/en/latest/pylasu.model.html#pylasu.model.model.Node) -[[Tylasu]](https://strumenta.github.io/tylasu/classes/core.node.html) - -Given we support references, the AST is actually a graph. However, references are usually rarer than containment relations, -and the parent-child relationship between nodes is the most common path for traversal, so the tree aspect is dominant. - -Suppose we have a Node representing a Java method declaration. -It may have the following properties: - -- A name attribute, of type String, with multiplicity 1..1 -- A reference to a method that it overrides, without an associated name, and with a multiplicity 0..1 -- A containment relation with its arguments, with multiplicity 0..* -- A containment relation to an optional body, with multiplicity 0..1 - -## Attribute Values - -Attribute values can have one of these types: - -- string (e.g. String in Kotlin, str in Python) -- character, in those languages that have a character type -- boolean -- numeric types (including bignums in those languages that have them) -- date and time types (e.g., in Kotlin, LocalDate, LocalTime, LocalDateTime) -- enum types (where applicable). An enum has a list of possible values, each one with a name. - -## Derived Properties - -Some properties could be calculated, i.e., derived from other properties. They are typically not serialized, e.g. to -JSON, with the rationale that we can recompute them. - -These properties cannot represent containment, only references. To mark a property as derived, we mark it with `@Derived`. -[[Kolasu]](https://javadoc.io/doc/com.strumenta.kolasu/kolasu-core/latest/com/strumenta/kolasu/model/Derived.html) - -Currently, neither Tylasu nor Pylasu supports this. - -## References by Name - -In textual languages we have typically references resolved by name (see [Naming](https://github.com/Strumenta/StarLasu/blob/main/documentation/naming.md)). For this reason in StarLasu we have special support for the type ReferenceByName. -This type has a generic parameter type, which should extend PossiblyNamed. This type indicates the type of thing that can be referred to. - -Suppose to have a language with the Node Type ReferenceExpression. The Node Type ReferenceExpression could have an attribute with type ReferenceByName<VariableDeclaration>, as it permits to refer to Variable Declarations. We would expect VariableDeclaration to extend PossiblyNamed (or Named). - -Note that we can accept to refer to Node Type which could potentially be anonymous, however we would typically only be able to refer to instances which actually have a name. Imagine a language that permits to define Classes and anonymous Classes. The Node Type ClassDeclaration would be PossiblyNamed, however when using a class in the typical class instantiation we would not be able to refer to anonymous classes but only to classes with a name. - -It has two values: - -- a name value, of type String -- a referred value - -The referred value is typically null initially and it is later set to a proper value during symbol resolution. - -## Error nodes - -Special AST nodes can be used to represent errors. - -All nodes representing errors should implement the same interface called `ErrorNode`. -[[Kolasu]](https://javadoc.io/doc/com.strumenta.kolasu/kolasu-core/latest/com/strumenta/kolasu/model/ErrorNode.html) -[[Pylasu]](https://pylasu.readthedocs.io/en/latest/pylasu.model.html#pylasu.model.errors.ErrorNode) -[[Tylasu]](https://strumenta.github.io/tylasu/classes/core.errornode.html) - -## Notes on Kolasu implementation - -### Link - -When a reference has no associated string, the property is marked as _Link_. - -For example: -```kotlin -class MethodDeclaration { - @Link val overriddenMethod: MethodDeclaration? -} - -``` - -### NodeType - -The interface `NodeType` can be used to specify that a certain Class represent a Node. This is intended to be used on -abstract classes that do not directly extend Node or interfaces. All concrete classes deriving from them should also -extend Node, indirectly. - -For example, if we define: - -```kotlin -@NodeType -interface Declaration -``` - -and later we have a Node using it: - -```kotlin -class CompilationUnit { - val elements: List // we know this is a containment, because elements are expected to be Nodes -} -``` - -We could then have different classes implementing Declaration: - -```kotlin -sealed class Statement: Node - -class VarDeclaration : Statement, Declaration // VarDeclaration indirectly extend Node -``` - diff --git a/documentation/cli_tools.md b/documentation/cli_tools.md deleted file mode 100644 index e221415..0000000 --- a/documentation/cli_tools.md +++ /dev/null @@ -1,15 +0,0 @@ -# Command Line Tools - -StarLasu supports the definition of CLI tools for each parser. These CLI tools are intended for the parser users. - -These tools permit to: - -- Generate and serialize the AST in various formats -- Generate statistics - -In addition to that, a series of tools for supporting parser developers. These are part of a separate (private) project named [StarLasu tools](https://github.com/Strumenta/starlasu-tools). - -_See in [Kolasu](https://github.com/Strumenta/kolasu/tree/master/core/src/main/kotlin/com/strumenta/kolasu/cli)_ - -_This is not yet supported in Pylasu and Tylasu._ - diff --git a/documentation/code_generation.md b/documentation/code_generation.md deleted file mode 100644 index 95cf841..0000000 --- a/documentation/code_generation.md +++ /dev/null @@ -1,4 +0,0 @@ -# Code generation - -Support for code generation is currently being worked on. It is planned to be included in Kolasu 1.6 and successively in Tylasu and Pylasu. - diff --git a/documentation/debug_print_format.md b/documentation/debug_print_format.md deleted file mode 100644 index 0c87948..0000000 --- a/documentation/debug_print_format.md +++ /dev/null @@ -1,40 +0,0 @@ -# Debug Print Format - -This functionality permits to print as a String an AST, in a format easy to inspect for a human. - -For example, given this AST: `Add(Add(Number(3), Number(9)), Number(1))` - -We could produce this string: -``` -Add { - left = [ - Add { - left = [ - Number { - value = 3 - } // Number - ] - right = [ - Number { - value = 9 - } // Number - ] - } // Add - ] - right = [ - Number { - value = 1 - } // Number - ] -} // Add -``` - -We can use a `DebugPrintConfiguration` instance to control which elements to filter, if to print empty lists, etc. - - -_This feature is present in Kolasu but it will not be necessarily replicated in all libraries_. - -_See in [Kolasu](https://github.com/Strumenta/kolasu/blob/master/core/src/main/kotlin/com/strumenta/kolasu/model/Printing.kt)_ - -_This is not yet supported in Pylasu._ - diff --git a/documentation/language_module.md b/documentation/language_module.md deleted file mode 100644 index 486677d..0000000 --- a/documentation/language_module.md +++ /dev/null @@ -1,7 +0,0 @@ -# Language Module - -A LanguageModule is an object that can both parse code into an AST and generate code from an AST. -Basically it is the combination of a parser and a code generator operating on the same node types. - -_See in [Kolasu](https://github.com/Strumenta/kolasu/blob/master/core/src/main/kotlin/com/strumenta/kolasu/language/LanguageModule.kt)_ - diff --git a/documentation/origin_and_destination.md b/documentation/origin_and_destination.md deleted file mode 100644 index 2bd3b34..0000000 --- a/documentation/origin_and_destination.md +++ /dev/null @@ -1,16 +0,0 @@ -# Origin and Destination - -A Node can have an _Origin_. An Origin indicates where the nodes come from. - -Let's see some examples: -- In the case of Nodes built by a parser, the Origin will indicate a point in some source file. -- In the case of Nodes built by a transformer, the Origin may indicate some Node which provided the original information. - -Consider this example: we are building a transpiler from RPG to Java. - -We first build an RPG parser that given RPG code product an RPG AST. The root of the AST may have type RPGCompilationUnit. The origin of an RPGCompilationUnit will represent the RPG file name and the position in that file. In the case of the root a position ranging from the first character of the first line, to the last character of the last line. - -Then we build a transformer that given an RPG AST produces a Java AST. The root of such AST may have type JavaCompilationUnit. The origin of a JavaCompilationUnit will refer to another node, in this case having type RPGCompilationUnit. - -In a Node we may also indicate what can be created from a Node. For example, if we build programmatically an AST and then we want to generate code from it, we may want to specify for each node what portion of code has been generated. For example, a node representing an If Statement in Java could end up being represented by a 5 lines of Java code inside a large Java file. In that case, the Destination will specify the name of the file and the lines and columns representing our If Statement. - diff --git a/documentation/playground.md b/documentation/playground.md deleted file mode 100644 index ff2339c..0000000 --- a/documentation/playground.md +++ /dev/null @@ -1,6 +0,0 @@ -# Strumenta Playground - -Strumenta Playground is a tool under development to visualize ASTs and transpilation traces. - -This is intended to help users of our parsers and transpilers to assess our work. - diff --git a/documentation/position.md b/documentation/position.md deleted file mode 100644 index b7e0090..0000000 --- a/documentation/position.md +++ /dev/null @@ -1,17 +0,0 @@ -# Position - -To indicate positions in textual files we use the class `Position`. - -A Position has a `start` and an `end` `Point`. Each position is relative to a certain `Source`, which is optional. - -A Point has a Line and a Column (both Int). The first line of a file is Line 1. The first Column of each line is 0. -The starting point of file has therefore Line=1 and Column=0. - -Important Source sub-classes are: - -* StringSource: represent an in-memory String -* FileSource: associated to a File -* URLSource: associated to an URL - -_See in [Kolasu](https://github.com/Strumenta/kolasu/blob/master/core/src/main/kotlin/com/strumenta/kolasu/model/Position.kt)_ - diff --git a/documentation/symbol_resolution.md b/documentation/symbol_resolution.md deleted file mode 100644 index de6aceb..0000000 --- a/documentation/symbol_resolution.md +++ /dev/null @@ -1,65 +0,0 @@ -# Symbol Resolution - -The objective of symbol resolution consists in linking name-based textual references to the corresponding node entity in the Abstract Syntax Tree (AST). StarLasu provides support for implementing such process with the following building blocks: - -* `PossiblyNamed` and `Named` interfaces can be implemented for nodes that can be referenced - see [Naming](https://github.com/Strumenta/StarLasu/blob/main/documentation/naming.md); -* `ReferenceByName` properties can be defined in nodes to represent links to other nodes; -* `SymbolResolver` instances can be configured to specify symbol resolution logic for each property or node type; -* `Scope` instances are used to resolve each reference in the AST; - -## Representing references among nodes - -References between nodes are implemented using `ReferenceByName` instances in StarLasu. These keep track of the relationship between a `name` and the `referred` node, which might be absent until the symbol resolution phase and must be a sub-type of `PossiblyNamed`. - -In [Kolasu](https://github.com/Strumenta/kolasu), for example, we can define a node `Person` containing a reference `friend` towards another `Person` as follows: -```kotlin -data class Person( - override val name: String, - val friend: ReferenceByName // <-- reference to another `Person` -) : PossiblyNamed -``` -Instances can then be created providing the `name` of the referred `Person` instance. As regards the actual referenced object, it might be provided as `initialReferred` value if known or left unresolved until symbol resolution. -```kotlin -// reference by name using `name` only -val first: Person = Person(friend = ReferenceByName("second")) -// reference by name using `initialReferred` value and `name` -val second: Person = Person(friend = ReferenceByName("first", first)) -``` -In general, references can be resolved using one or more candidates, as follows -```kotlin -second.tryToResolve(first) // <-- `first` is the only candidate -second.tryToResolve(listOf(first, second, others)) // <-- list of candidates -``` -While it is possible to manually implement symbol resolution by traversing the AST and updating the `referred` value for each `ReferenceByName` property, StarLasu provides support for the declarative specification of symbol resolution rules, as shown in the next section. - -## Declarative symbol resolution - -As mentioned in the previous section, it is surely possible to manually implement symbol resolution as some kind of tree-traversal algorithm. However, StarLasu provides support to ease such task and allows the developer to focus on language-specific concerns by providing rules for each reference in a given AST. - -Symbol resolution rule specifications consist of three parts: - -* __guard__ - the reference property for which we want to provide a scope; -* __context__ - the node from which we want to compute the scope; -* __body__ - the actual scope definition, i.e. `Scope` instance; - -Each rule produces a `Scope` instance that is used to resolve a given property. Given a property, StarLasu adopts a precise rule resolution schema. Considering `Person::friend`, for example, the following steps will be performed: - -1) lookup for a property-based rule having `Person::friend` as guard and `Person` as context; -2) lookup for a property-based rule having `Person::friend` as guard and any ancestor of the `Person` node as context; - -As soon as one rule is found, the symbol resolver will use it to resolve the reference. - -In our example, we could define that `friend` reference candidates should correspond to aggregating all `Person` instances contained in the `CompilationUnit` of the AST as follows: -```kotlin -val symbolResolver = symbolResolver { - // property-based rule for Person::friend property - scopeFor(Person::friend) { - scope { - it.findAncestorOfType(CompilationUnit::class.java) - ?.walk() - ?.filterIsInstance() - ?.forEach(this::define) - } - } -} -``` diff --git a/documentation/typesystem.md b/documentation/typesystem.md deleted file mode 100644 index 7740f7d..0000000 --- a/documentation/typesystem.md +++ /dev/null @@ -1,5 +0,0 @@ -# Typesystem - -Support for typesystem has not yet been implemented as part of StarLasu. It has been implemented in several parsers built using StarLasu -but it has yet to be ported to StarLasu itself. - diff --git a/generatedoc.sh b/generatedoc.sh deleted file mode 100755 index 3be161c..0000000 --- a/generatedoc.sh +++ /dev/null @@ -1,15 +0,0 @@ -HTMLOUTPUT="build/htmldoc/StarLasuOverview.html" -PDFOUTPUT="build/pdfdoc/StarLasuOverview.pdf" - -sh ./preparedoc.sh - -pandoc -c css/documentation.css -s build/source.md --metadata pagetitle="StarLasu Overview" -o $HTMLOUTPUT - -pandoc -c css/documentation.css -H configuration/listings-setup.tex --variable colorlinks=true build/source.md --listings --metadata pagetitle="StarLasu Overview" -o $PDFOUTPUT - - - - - - - diff --git a/preparedoc.sh b/preparedoc.sh deleted file mode 100644 index 406df28..0000000 --- a/preparedoc.sh +++ /dev/null @@ -1,36 +0,0 @@ -mkdir -p build/htmldoc -mkdir -p build/pdfdoc - -cp -R css build/htmldoc - -cat documentation/README.md documentation/usecases/building-parser.md documentation/usecases/building-transpiler.md documentation/usecases/building-codegenerator.md documentation/ast_definition.md documentation/position.md documentation/origin_and_destination.md documentation/traversing.md documentation/transformations.md documentation/debug_print_format.md documentation/serialization.md documentation/naming.md documentation/symbol_resolution.md documentation/typesystem.md documentation/code_generation.md documentation/language_module.md documentation/ast_common_elements.md documentation/cli_tools.md documentation/parsetree_to_ast.md documentation/validation.md documentation/testing.md documentation/emf.md documentation/playground.md documentation/parsers-cross-platform.md > build/source.md - -echo "PREPARE DOC - INITIAL SOURCE CREATED" - -gsed -i 's/https:\/\/github.com\/Strumenta\/StarLasu\/blob\/main\/documentation\/usecases\///g' build/source.md -gsed -i 's/https:\/\/github.com\/Strumenta\/StarLasu\/blob\/main\/documentation\///g' build/source.md -gsed -i 's/usecases\/building-parser.md/#building-a-parser/g' build/source.md -gsed -i 's/usecases\/building-transpiler.md/#building-a-transpiler/g' build/source.md -gsed -i 's/usecases\/building-codegenerator.md/#building-a-code-generator/g' build/source.md -gsed -i 's/ast_definition.md/#ast-definition/g' build/source.md -gsed -i 's/position.md/#position/g' build/source.md -gsed -i 's/origin_and_destination.md/#origin-and-destination/g' build/source.md -gsed -i 's/traversing.md/#traversing-the-ast/g' build/source.md -gsed -i 's/transformations.md/#transforming-asts/g' build/source.md -gsed -i 's/debug_print_format.md/#debug-print-format/g' build/source.md -gsed -i 's/serialization.md/#serialization/g' build/source.md -gsed -i 's/naming.md/#naming/g' build/source.md -gsed -i 's/symbol_resolution.md/#symbol-resolution/g' build/source.md -gsed -i 's/typesystem.md/#typesystem/g' build/source.md -gsed -i 's/code_generation.md/#code-generation/g' build/source.md -gsed -i 's/language_module.md/#language-module/g' build/source.md -gsed -i 's/ast_common_elements.md/#ast-common-elements/g' build/source.md -gsed -i 's/cli_tools.md/#command-line-tools/g' build/source.md -gsed -i 's/parsetree_to_ast.md/#parse-tree-to-ast-mapping/g' build/source.md -gsed -i 's/validation.md/#validation/g' build/source.md -gsed -i 's/testing.md/#testing/g' build/source.md -gsed -i 's/emf.md/#emf/g' build/source.md -gsed -i 's/playground.md/#strumenta-playground/g' build/source.md -gsed -i 's/parsers-cross-platform.md/#cross-platform-parsers/g' build/source.md - -echo "PREPARE DOC - END" \ No newline at end of file