gbf @ 93188c5

M gleam.toml

@@ -14,6 +14,7 @@ # https://gleam.run/writing-gleam/gleam-toml/.
 
 [dependencies]
 gleam_stdlib = ">= 0.44.0 and < 2.0.0"
+splitter = ">= 1.1.0 and < 2.0.0"
 
 [dev-dependencies]
 gleeunit = ">= 1.0.0 and < 2.0.0"

M manifest.toml

@@ -4,8 +4,10 @@
 packages = [
   { name = "gleam_stdlib", version = "0.65.0", build_tools = ["gleam"], requirements = [], otp_app = "gleam_stdlib", source = "hex", outer_checksum = "7C69C71D8C493AE11A5184828A77110EB05A7786EBF8B25B36A72F879C3EE107" },
   { name = "gleeunit", version = "1.6.1", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleeunit", source = "hex", outer_checksum = "FDC68A8C492B1E9B429249062CD9BAC9B5538C6FBF584817205D0998C42E1DAC" },
+  { name = "splitter", version = "1.1.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "splitter", source = "hex", outer_checksum = "05564A381580395DCDEFF4F88A64B021E8DAFA6540AE99B4623962F52976AA9D" },
 ]
 
 [requirements]
 gleam_stdlib = { version = ">= 0.44.0 and < 2.0.0" }
 gleeunit = { version = ">= 1.0.0 and < 2.0.0" }
+splitter = { version = ">= 1.1.0 and < 2.0.0" }

A src/gbf/lexer.gleam

@@ -0,0 +1,81 @@
+import gbf/token.{type Token}
+import gleam/list
+import gleam/string
+import splitter
+
+pub opaque type Lexer {
+  Lexer(source: String, offset: Int, newlines: splitter.Splitter)
+}
+
+pub type Position {
+  Position(offset: Int)
+}
+
+pub fn new(source) {
+  Lexer(source:, offset: 0, newlines: splitter.new(["\r\n", "\n"]))
+}
+
+pub fn lex(lexer: Lexer) -> List(#(Token, Position)) {
+  do_lex(lexer, [])
+  |> list.reverse
+}
+
+fn do_lex(lexer: Lexer, tokens: List(#(Token, Position))) {
+  case next(lexer) {
+    #(_, #(token.EOF, _)) -> tokens
+    #(lexer, token) -> do_lex(lexer, [token, ..tokens])
+  }
+}
+
+fn next(lexer: Lexer) {
+  case lexer.source {
+    " " <> source | "\n" <> source | "\r" <> source | "\t" <> source ->
+      advance(lexer, source, 1) |> next
+
+    ">" <> source -> token(lexer, token.IncrementPointer, source, 1)
+    "<" <> source -> token(lexer, token.DecrementPointer, source, 1)
+    "+" <> source -> token(lexer, token.IncrementByte, source, 1)
+    "-" <> source -> token(lexer, token.DecrementByte, source, 1)
+    "." <> source -> token(lexer, token.OutputByte, source, 1)
+    "," <> source -> token(lexer, token.InputByte, source, 1)
+    "[" <> source -> token(lexer, token.StartBlock, source, 1)
+    "]" <> source -> token(lexer, token.EndBlock, source, 1)
+
+    _ ->
+      case string.pop_grapheme(lexer.source) {
+        Error(_) -> #(lexer, #(token.EOF, Position(lexer.offset)))
+        Ok(_) -> comment(lexer, lexer.offset)
+      }
+  }
+}
+
+fn advance(lexer, source, offset) {
+  Lexer(..lexer, source:, offset: lexer.offset + offset)
+}
+
+fn advanced(
+  token: #(Token, Position),
+  lexer: Lexer,
+  source: String,
+  offset: Int,
+) -> #(Lexer, #(Token, Position)) {
+  #(advance(lexer, source, offset), token)
+}
+
+fn token(
+  lexer: Lexer,
+  token: Token,
+  source: String,
+  offset: Int,
+) -> #(Lexer, #(Token, Position)) {
+  #(token, Position(offset: lexer.offset))
+  |> advanced(lexer, source, offset)
+}
+
+fn comment(lexer: Lexer, start: Int) -> #(Lexer, #(Token, Position)) {
+  let #(prefix, suffix) = splitter.split_before(lexer.newlines, lexer.source)
+  let eaten = string.byte_size(prefix)
+  let lexer = advance(lexer, suffix, eaten)
+
+  #(lexer, #(token.Comment(prefix), Position(start)))
+}

M src/gbf/token.gleam

@@ -1,6 +1,6 @@
 pub type Token {
   /// Everything that is not one of the tokens below, considered the be a comment
-  Comment
+  Comment(String)
 
   /// Increment the data pointer by one (to point to the next cell to the right).
   /// `>` symbol

A test/gbf_lexer_test.gleam

@@ -0,0 +1,37 @@
+import gbf/lexer
+import gbf/token
+import gleeunit/should
+
+pub fn can_lex_test() {
+  "><+-.,[] this is a comment"
+  |> lexer.new
+  |> lexer.lex
+  |> should.equal([
+    #(token.IncrementPointer, lexer.Position(0)),
+    #(token.DecrementPointer, lexer.Position(1)),
+    #(token.IncrementByte, lexer.Position(2)),
+    #(token.DecrementByte, lexer.Position(3)),
+    #(token.OutputByte, lexer.Position(4)),
+    #(token.InputByte, lexer.Position(5)),
+    #(token.StartBlock, lexer.Position(6)),
+    #(token.EndBlock, lexer.Position(7)),
+    #(token.Comment("this is a comment"), lexer.Position(9)),
+  ])
+}
+
+pub fn multiline_test() {
+  "this is a comment
++++
+<.
+  "
+  |> lexer.new
+  |> lexer.lex
+  |> should.equal([
+    #(token.Comment("this is a comment"), lexer.Position(0)),
+    #(token.IncrementByte, lexer.Position(18)),
+    #(token.IncrementByte, lexer.Position(19)),
+    #(token.IncrementByte, lexer.Position(20)),
+    #(token.DecrementPointer, lexer.Position(22)),
+    #(token.OutputByte, lexer.Position(23)),
+  ])
+}

M test/gbf_test.gleam

@@ -3,11 +3,3 @@
 pub fn main() -> Nil {
   gleeunit.main()
 }
-
-// gleeunit test functions end in `_test`
-pub fn hello_world_test() {
-  let name = "Joe"
-  let greeting = "Hello, " <> name <> "!"
-
-  assert greeting == "Hello, Joe!"
-}