diff --git a/.github/scripts/check_newline_preservation.sh b/.github/scripts/check_newline_preservation.sh new file mode 100644 index 00000000..620b9da8 --- /dev/null +++ b/.github/scripts/check_newline_preservation.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# +# Checks that formatting a file keeps the line endings it came with. The +# detection is exercised through the binary rather than the library, because +# reading the file, detecting the ending and writing it back out are three +# separate steps and any one of them can drop it. +# +# Usage: bash ./.github/scripts/check_newline_preservation.sh + +set -euo pipefail + +BINARY=$(realpath "${1:?usage: check_newline_preservation.sh }") +SOURCE_FILE="examples/jbeam/frame.jbeam" + +WORK_DIR=$(mktemp -d) +trap 'rm -rf "$WORK_DIR"' EXIT + +# The binary runs from the work directory, so point it back at the repo for the +# default ruleset it ships as a data file. +export jbeam_edit_datadir="$PWD" + +failed=0 + +carriage_returns() { + grep -c $'\r' "$1" || true +} + +check() { + local label="$1" expected="$2" target="$WORK_DIR/$1.jbeam" + + if [[ "$label" == crlf ]]; then + sed 's/\r$//; s/$/\r/' "$SOURCE_FILE" >"$target" + else + sed 's/\r$//' "$SOURCE_FILE" >"$target" + fi + + local before after + before=$(carriage_returns "$target") + if [[ "$before" -ne "$expected" ]]; then + echo "setup error: $label input has $before carriage returns, expected $expected" + failed=1 + return + fi + + (cd "$WORK_DIR" && "$BINARY" "$label.jbeam" >/dev/null) + + after=$(carriage_returns "$target") + if [[ "$expected" -eq 0 && "$after" -ne 0 ]]; then + echo "$label: line endings changed, LF input came out with $after carriage returns" + failed=1 + elif [[ "$expected" -ne 0 && "$after" -eq 0 ]]; then + echo "$label: line endings changed, CRLF input came out with no carriage returns" + failed=1 + fi +} + +lf_lines=$(sed 's/\r$//' "$SOURCE_FILE" | wc -l) + +check lf 0 +check crlf "$lf_lines" + +if [[ "$failed" -ne 0 ]]; then + echo "formatting does not preserve line endings" + exit 1 +fi + +echo "line endings survive a format for both LF and CRLF input" diff --git a/.github/scripts/prepare_installer.sh b/.github/scripts/prepare_installer.sh index fce5227f..c9cf0fd1 100644 --- a/.github/scripts/prepare_installer.sh +++ b/.github/scripts/prepare_installer.sh @@ -49,8 +49,11 @@ cp ./examples/jbeam-edit.yaml ./.jbeam-edit.yaml JBEAM_DIR="./examples/jbeam" +# Line endings are checked separately by check_newline_preservation.sh. Here we +# only care about content, and the two sides can disagree on endings because the +# input comes from a blob while the expected file comes from the working tree. custom_diff() { - diff --color=always --suppress-common-lines "$1" "$2" + diff --color=always --suppress-common-lines --strip-trailing-cr "$1" "$2" } mapfile -t JBEAM_FILES < <(find "$JBEAM_DIR" -maxdepth 1 -name "*.jbeam" -printf "%f\n") diff --git a/.github/workflows/build-and-test.yaml b/.github/workflows/build-and-test.yaml index 9bfff6b2..42196181 100644 --- a/.github/workflows/build-and-test.yaml +++ b/.github/workflows/build-and-test.yaml @@ -198,3 +198,8 @@ jobs: run: cabal build --project-file cabal.project.ci all - name: Run tests (GHC ${{ steps.setup-ghc.outputs.ghc-version }}) run: cabal test --project-file cabal.project.ci + - name: Check that formatting preserves line endings + shell: bash + run: | + bash ./.github/scripts/check_newline_preservation.sh \ + "$(cabal list-bin --project-file cabal.project.ci jbeam-edit)" diff --git a/exe/jbeam-edit/Main.hs b/exe/jbeam-edit/Main.hs index c65c9e25..f037284d 100644 --- a/exe/jbeam-edit/Main.hs +++ b/exe/jbeam-edit/Main.hs @@ -5,6 +5,7 @@ module Main ( import CommandLineOptions import Control.Monad (when) import Data.Text (Text) +import JbeamEdit.Core.Newline import JbeamEdit.Core.Node (Node) import JbeamEdit.Formatting (RuleSet, formatNodeAndWrite) import JbeamEdit.Formatting.Config @@ -12,12 +13,9 @@ import JbeamEdit.IOUtils import JbeamEdit.Parsing.Jbeam (parseNodes) import System.Directory.OsPath import System.Environment (getArgs) +import System.IO (Newline) import System.OsPath -#ifdef ENABLE_WINDOWS_NEWLINES -import Data.Text qualified as T -#endif - #ifdef ENABLE_TRANSFORMATION import JbeamEdit.Transformation import JbeamEdit.Transformation.Config @@ -48,16 +46,19 @@ editFile opts = do Just filename -> do createBackupFile filename opts contents <- tryReadFile [] filename - case contents >>= parseNodes of - Right ns -> processNodes opts filename ns formattingConfig + case contents of + Right contents' -> + case parseNodes contents' of + Right ns -> processNodes (detectNewline contents') opts filename ns formattingConfig + Left err -> putErrorLine err Left err -> putErrorLine err Nothing -> putErrorLine "missing arg filename" -processNodes :: Options -> OsPath -> Node -> RuleSet -> IO () -processNodes opts outFile nodes formattingConfig = do +processNodes :: Newline -> Options -> OsPath -> Node -> RuleSet -> IO () +processNodes newline opts outFile nodes formattingConfig = do transformedNode <- applyTransform formattingConfig opts nodes case transformedNode of - Right transformedNode' -> formatNodeAndWrite formattingConfig outFile transformedNode' + Right transformedNode' -> formatNodeAndWrite newline formattingConfig outFile transformedNode' Left err -> putErrorLine err applyTransform :: RuleSet -> Options -> Node -> IO (Either Text Node) diff --git a/jbeam-edit.cabal b/jbeam-edit.cabal index a60dfd62..c101eea4 100644 --- a/jbeam-edit.cabal +++ b/jbeam-edit.cabal @@ -11,7 +11,7 @@ license-file: LICENSE copyright: 2025 webdevred maintainer: example@example.com author: webdevred -tested-with: ghc ==9.10.3 ghc ==9.14.1 +tested-with: ghc ==9.8.4 ghc ==9.10.3 ghc ==9.14.1 homepage: https://github.com/webdevred/jbeam-edit#readme bug-reports: https://github.com/webdevred/jbeam-edit/issues synopsis: @@ -63,6 +63,7 @@ flag windows-example-paths library exposed-modules: + JbeamEdit.Core.Newline JbeamEdit.Core.Node JbeamEdit.Core.NodeCursor JbeamEdit.Core.NodePath @@ -118,9 +119,6 @@ library if (os(windows) && flag(windows-example-paths)) cpp-options: -DWINDOWS_EXAMPLE_PATHS - if os(windows) - cpp-options: -DENABLE_WINDOWS_NEWLINES - library jbeam-edit-transformation exposed-modules: JbeamEdit.Transformation @@ -262,6 +260,7 @@ test-suite jbeam-edit-test build-tool-depends: hspec-discover:hspec-discover hs-source-dirs: test other-modules: + Core.NewlineSpec Core.NodeCursorSpec Core.NodePathSpec Core.NodeSpec diff --git a/package.yaml b/package.yaml index e07ccea6..04f0a267 100644 --- a/package.yaml +++ b/package.yaml @@ -25,7 +25,7 @@ description: >- and can automatically rename nodes and update references. Custom formatting rules are supported via JBFL (JBeam Formatting Language). See the README for usage instructions and examples: https://github.com/webdevred/jbeam-edit#readme -tested-with: [GHC == 9.10.3, GHC == 9.14.1] +tested-with: [GHC == 9.8.4, GHC == 9.10.3, GHC == 9.14.1] dependencies: - base >= 4.19 @@ -101,8 +101,6 @@ library: cpp-options: -DENABLE_TRANSFORMATION - condition: os(windows) && flag(windows-example-paths) cpp-options: -DWINDOWS_EXAMPLE_PATHS - - condition: os(windows) - cpp-options: -DENABLE_WINDOWS_NEWLINES _jbeam-options: &jbeam-options main: Main.hs diff --git a/src-extra/transformation/JbeamEdit/Transformation.hs b/src-extra/transformation/JbeamEdit/Transformation.hs index 8e043502..081e5142 100644 --- a/src-extra/transformation/JbeamEdit/Transformation.hs +++ b/src-extra/transformation/JbeamEdit/Transformation.hs @@ -5,7 +5,7 @@ import Data.Bifunctor (first) import Data.Bool (bool) import Data.Foldable.Extra (notNull) import Data.Function (on) -import Data.List (partition) +import Data.List (foldl', partition) import Data.List.NonEmpty (NonEmpty) import Data.List.NonEmpty qualified as NE import Data.Map (Map) @@ -24,6 +24,7 @@ import Data.Traversable (mapAccumL) import Data.Vector (Vector, (!), (!?), (//)) import Data.Vector qualified as V import GHC.IsList +import JbeamEdit.Core.Newline import JbeamEdit.Core.Node import JbeamEdit.Core.NodeCursor (newCursor) import JbeamEdit.Core.NodeCursor qualified as NC @@ -492,12 +493,15 @@ filterJbeamFiles excludedFilenames = filter go updateOtherFiles :: RuleSet -> UpdateNamesMap -> OsPath -> IO () updateOtherFiles formattingConfig updatedNames filepath = do contents <- tryReadFile [] filepath - case contents >>= parseNodes of - Right node -> - let node' = findAndUpdateTextInNode updatedNames newCursor node - in when - (node /= node') - (formatNodeAndWrite formattingConfig filepath node') + case contents of + Right contents' -> + case parseNodes contents' of + Right node -> + let node' = findAndUpdateTextInNode updatedNames newCursor node + in when + (node /= node') + (formatNodeAndWrite (detectNewline contents') formattingConfig filepath node') + Left err -> putErrorLine err Left err -> putErrorLine err transform diff --git a/src-extra/transformation/JbeamEdit/Transformation/BeamValidation.hs b/src-extra/transformation/JbeamEdit/Transformation/BeamValidation.hs index f0c3aee3..e4e34df3 100644 --- a/src-extra/transformation/JbeamEdit/Transformation/BeamValidation.hs +++ b/src-extra/transformation/JbeamEdit/Transformation/BeamValidation.hs @@ -7,6 +7,7 @@ module JbeamEdit.Transformation.BeamValidation ( ) where import Control.Monad (forM_, unless) +import Data.List (foldl') import Data.List.NonEmpty (toList) import Data.Map qualified as M import Data.Set (Set) diff --git a/src/JbeamEdit/Core/Newline.hs b/src/JbeamEdit/Core/Newline.hs new file mode 100644 index 00000000..b1a43020 --- /dev/null +++ b/src/JbeamEdit/Core/Newline.hs @@ -0,0 +1,15 @@ +module JbeamEdit.Core.Newline (detectNewline) where + +import Data.ByteString.Lazy (LazyByteString) +import Data.ByteString.Lazy qualified as LBS +import Data.ByteString.Lazy.Char8 qualified as LBS8 (elemIndex) +import System.IO (Newline (..)) + +detectNewline :: LazyByteString -> Newline +detectNewline content = + case LBS8.elemIndex '\r' content of + Nothing -> LF + Just idx -> + case LBS.uncons (LBS.drop (idx + 1) content) of + Just (10, _) -> CRLF -- '\n' + _ -> detectNewline (LBS.drop (idx + 1) content) diff --git a/src/JbeamEdit/Formatting.hs b/src/JbeamEdit/Formatting.hs index 320f0a06..34ebabb2 100644 --- a/src/JbeamEdit/Formatting.hs +++ b/src/JbeamEdit/Formatting.hs @@ -52,6 +52,7 @@ import JbeamEdit.Formatting.Rules ( import JbeamEdit.Formatting.Rules.ComplexNewLine qualified as CNL import JbeamEdit.Formatting.Rules.TrailingComma qualified as TC import System.File.OsPath qualified as OS (writeFile) +import System.IO (Newline (..)) import System.OsPath (OsPath) data FormattingState = FormattingState @@ -459,22 +460,23 @@ formatWithCursor rs _ cursor n = formatNode :: RuleSet -> Node -> Text formatNode rs node = formatWithCursor rs emptyState newCursor node <> T.singleton '\n' -#ifdef ENABLE_WINDOWS_NEWLINES -replaceNewlines :: Text -> Text -replaceNewlines = T.replace "\n" "\r\n" -#else -replaceNewlines :: Text -> Text -replaceNewlines = id -#endif +{- | 'formatNode' always emits LF, so rewrite the line endings when the file +came with CRLF. The handle's newline mode cannot do this, it only applies to +text-mode writes and the output goes out as bytes. +-} +applyNewline :: Newline -> Text -> Text +applyNewline CRLF = T.replace "\n" "\r\n" +applyNewline LF = id formatNodeAndWrite - :: RuleSet + :: Newline + -> RuleSet -> OsPath -> Node -> IO () -formatNodeAndWrite rs outFile = +formatNodeAndWrite newline rs outFile = OS.writeFile outFile . LBS.fromStrict . encodeUtf8 - . replaceNewlines + . applyNewline newline . formatNode rs diff --git a/src/JbeamEdit/Parsing/Jbeam.hs b/src/JbeamEdit/Parsing/Jbeam.hs index a4fa95b9..be8784e9 100644 --- a/src/JbeamEdit/Parsing/Jbeam.hs +++ b/src/JbeamEdit/Parsing/Jbeam.hs @@ -98,7 +98,8 @@ associationDirection st = bool PreviousNode NextNode (lastNodeEndedWithNewline s commentStripSpace :: Text -> Text commentStripSpace initialText = - let initialNewline = mwhen (T.isPrefixOf "\n" initialText) "\n" + let startsWithNewline text = T.isPrefixOf "\n" text || T.isPrefixOf "\r\n" text + initialNewline = mwhen (startsWithNewline initialText) "\n" trimTrailingSpaces = T.dropWhileEnd (charBoth (/= '\n') isSpace) endingNewline = mwhen (T.isSuffixOf "\n" $ trimTrailingSpaces initialText) "\n" go = T.intercalate "\n" . filter (not . T.all isSpace) . map T.strip . T.lines diff --git a/test/Core/NewlineSpec.hs b/test/Core/NewlineSpec.hs new file mode 100644 index 00000000..59bb68fd --- /dev/null +++ b/test/Core/NewlineSpec.hs @@ -0,0 +1,17 @@ +module Core.NewlineSpec (spec) where + +import JbeamEdit.Core.Newline (detectNewline) +import System.IO (Newline (..)) +import Test.Hspec + +spec :: Spec +spec = + describe "detectNewline" $ do + it "detects LF for unix line endings" $ + detectNewline "a\nb\n" `shouldBe` LF + it "detects CRLF for windows line endings" $ + detectNewline "a\r\nb\r\n" `shouldBe` CRLF + it "detects CRLF when line endings are mixed" $ + detectNewline "a\r\nb\n" `shouldBe` CRLF + it "defaults to LF when there is no line ending" $ + detectNewline "abc" `shouldBe` LF