From 359b1467a29895747adb55d053d0785782547a89 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 24 Oct 2018 20:33:52 -0700 Subject: [PATCH 001/763] Work around snap's old cabal + new snapcraft proxy. --- .snapsquid.conf | 14 ++++++++++++++ snap/snapcraft.yaml | 9 ++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 .snapsquid.conf diff --git a/.snapsquid.conf b/.snapsquid.conf new file mode 100644 index 0000000..205c1a6 --- /dev/null +++ b/.snapsquid.conf @@ -0,0 +1,14 @@ +# In 2015, cabal-install had a http bug triggered when proxies didn't keep +# the connection open. This version made it into Ubuntu Xenial as used by +# Snapcraft. In June 2018, Snapcraft's proxy started triggering this bug. +# +# https://bugs.launchpad.net/launchpad-buildd/+bug/1797809 +# +# Workaround: add more proxy + +visible_hostname localhost +http_port 8888 +cache_peer 10.10.10.1 parent 8222 0 no-query default +cache_peer_domain localhost !.internal +http_access allow all + diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index b7f0e96..9c50293 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -37,9 +37,16 @@ parts: source: ./ build-packages: - cabal-install + - squid3 build: | + # See comments in .snapsquid.conf + [ "$http_proxy" ] && { + squid3 -f .snapsquid.conf + export http_proxy="http://localhost:8888" + sleep 3 + } cabal sandbox init - cabal update + cabal update || cat /var/log/squid/* cabal install -j install: | install -d $SNAPCRAFT_PART_INSTALL/usr/bin From 620c9c20232468cd1e8d06a18e45336321609961 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 1 Nov 2018 04:47:44 -0700 Subject: [PATCH 002/763] Also warn about glob matching with [ a != b* ] (fixes #1374) --- src/ShellCheck/Analytics.hs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 713a298..3002e3e 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1197,11 +1197,12 @@ prop_checkComparisonAgainstGlob2 = verifyNot checkComparisonAgainstGlob "[[ $cow prop_checkComparisonAgainstGlob3 = verify checkComparisonAgainstGlob "[ $cow = *foo* ]" prop_checkComparisonAgainstGlob4 = verifyNot checkComparisonAgainstGlob "[ $cow = foo ]" prop_checkComparisonAgainstGlob5 = verify checkComparisonAgainstGlob "[[ $cow != $bar ]]" +prop_checkComparisonAgainstGlob6 = verify checkComparisonAgainstGlob "[ $f != /* ]" checkComparisonAgainstGlob _ (TC_Binary _ DoubleBracket op _ (T_NormalWord id [T_DollarBraced _ _])) | op `elem` ["=", "==", "!="] = warn id 2053 $ "Quote the right-hand side of " ++ op ++ " in [[ ]] to prevent glob matching." checkComparisonAgainstGlob _ (TC_Binary _ SingleBracket op _ word) - | (op == "=" || op == "==") && isGlob word = + | op `elem` ["=", "==", "!="] && isGlob word = err (getId word) 2081 "[ .. ] can't match globs. Use [[ .. ]] or case statement." checkComparisonAgainstGlob _ _ = return () From 5e1b1e010a82599e4f83c48c4656deff7b3cd7e0 Mon Sep 17 00:00:00 2001 From: Peter Dave Hello Date: Fri, 2 Nov 2018 13:47:22 +0800 Subject: [PATCH 003/763] Update Docker build-only image to Ubuntu 18.04 Ref: > Ubuntu 17.10 (Artful Aardvark) End of Life reached on July 19 2018 https://fridge.ubuntu.com/2018/07/19/ubuntu-17-10-artful-aardvark-end-of-life-reached-on-july-19-2018/ --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 5ecfd08..2b65291 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build-only image -FROM ubuntu:17.10 AS build +FROM ubuntu:18.04 AS build USER root WORKDIR /opt/shellCheck From 2827b35696bb54c16f50aac0bea91cc65e2d11c6 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 7 Nov 2018 17:57:07 -0800 Subject: [PATCH 004/763] SC2240: Warn about `. script args..` in sh/dash (fixes #1373) --- CHANGELOG.md | 1 + src/ShellCheck/Checks/Commands.hs | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 60f890c..dc6d258 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ - SC2236/SC2237: Suggest -n/-z instead of ! -z/-n - SC2238: Warn when redirecting to a known command name, e.g. ls > rm - SC2239: Warn if the shebang is not an absolute path, e.g. #!bin/sh +- SC2240: Warn shen passing additional arguments to dot (.) in sh/dash - SC1133: Better diagnostics when starting a line with |/||/&& ### Changed - Most warnings now have useful end positions diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 2c8b00a..fbbb67a 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -92,6 +92,7 @@ commandChecks = [ ,checkWhich ,checkSudoRedirect ,checkSudoArgs + ,checkSourceArgs ] buildCommandMap :: [CommandCheck] -> Map.Map CommandName (Token -> Analysis) @@ -1008,5 +1009,16 @@ checkSudoArgs = CommandCheck (Basename "sudo") f -- This mess is why ShellCheck prefers not to know. parseOpts = getBsdOpts "vAknSbEHPa:g:h:p:u:c:T:r:" +prop_checkSourceArgs1 = verify checkSourceArgs "#!/bin/sh\n. script arg" +prop_checkSourceArgs2 = verifyNot checkSourceArgs "#!/bin/sh\n. script" +prop_checkSourceArgs3 = verifyNot checkSourceArgs "#!/bin/bash\n. script arg" +checkSourceArgs = CommandCheck (Exactly ".") f + where + f t = whenShell [Sh, Dash] $ + case arguments t of + (file:arg1:_) -> warn (getId arg1) 2240 $ + "The dot command does not support arguments in sh/dash. Set them as variables." + _ -> return () + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) From f4044fbcc7b5134f3cd63682afa9975097c7a90f Mon Sep 17 00:00:00 2001 From: Zero King Date: Thu, 8 Nov 2018 03:07:52 +0000 Subject: [PATCH 005/763] Fix typo in CHANGELOG --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dc6d258..22e3207 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ - SC2236/SC2237: Suggest -n/-z instead of ! -z/-n - SC2238: Warn when redirecting to a known command name, e.g. ls > rm - SC2239: Warn if the shebang is not an absolute path, e.g. #!bin/sh -- SC2240: Warn shen passing additional arguments to dot (.) in sh/dash +- SC2240: Warn when passing additional arguments to dot (.) in sh/dash - SC1133: Better diagnostics when starting a line with |/||/&& ### Changed - Most warnings now have useful end positions From e705552c970f5e338e277006f9b966a7ddec40ad Mon Sep 17 00:00:00 2001 From: Roman Zolotarev Date: Thu, 15 Nov 2018 08:49:26 +0000 Subject: [PATCH 006/763] Update README.html Add OpenBSD to "Installing" section. --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 3b2ee5e..ba53dbb 100644 --- a/README.md +++ b/README.md @@ -133,6 +133,10 @@ On OS X with homebrew: brew install shellcheck +On OpenBSD: + + pkg_add shellcheck + On openSUSE zypper in ShellCheck From cb76951ad23616b971493674eea0170864bddbc4 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 24 Nov 2018 22:56:22 -0800 Subject: [PATCH 007/763] Add warnings for 'exit' similar to 'return' (fixes #1388) --- CHANGELOG.md | 3 +++ src/ShellCheck/Checks/Commands.hs | 24 +++++++++++++++++++----- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 22e3207..947e456 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,14 +2,17 @@ ### Added - Command line option --severity/-S for filtering by minimum severity - Command line option --wiki-link-count/-W for showing wiki links +- SC2152/SC2151: Warn about bad `exit` values like `1234` and `"foo"` - SC2236/SC2237: Suggest -n/-z instead of ! -z/-n - SC2238: Warn when redirecting to a known command name, e.g. ls > rm - SC2239: Warn if the shebang is not an absolute path, e.g. #!bin/sh - SC2240: Warn when passing additional arguments to dot (.) in sh/dash - SC1133: Better diagnostics when starting a line with |/||/&& + ### Changed - Most warnings now have useful end positions - SC1117 about unknown double-quoted escape sequences has been retired + ### Fixed - SC2021 no longer triggers for equivalence classes like '[=e=]' - SC2221/SC2222 no longer mistriggers on fall-through case branches diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index fbbb67a..f4ead5b 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -61,6 +61,7 @@ commandChecks = [ ,checkGrepRe ,checkTrapQuotes ,checkReturn + ,checkExit ,checkFindExecWithSingleArgument ,checkUnusedEchoEscapes ,checkInjectableFindSh @@ -281,15 +282,28 @@ prop_checkReturn4 = verifyNot checkReturn "return $((a|b))" prop_checkReturn5 = verify checkReturn "return -1" prop_checkReturn6 = verify checkReturn "return 1000" prop_checkReturn7 = verify checkReturn "return 'hello world'" -checkReturn = CommandCheck (Exactly "return") (f . arguments) +checkReturn = CommandCheck (Exactly "return") (returnOrExit + (\c -> err c 2151 "Only one integer 0-255 can be returned. Use stdout for other data.") + (\c -> err c 2152 "Can only return 0-255. Other data should be written to stdout.")) + +prop_checkExit1 = verifyNot checkExit "exit" +prop_checkExit2 = verifyNot checkExit "exit 1" +prop_checkExit3 = verifyNot checkExit "exit $var" +prop_checkExit4 = verifyNot checkExit "exit $((a|b))" +prop_checkExit5 = verify checkExit "exit -1" +prop_checkExit6 = verify checkExit "exit 1000" +prop_checkExit7 = verify checkExit "exit 'hello world'" +checkExit = CommandCheck (Exactly "exit") (returnOrExit + (\c -> err c 2241 "The exit status can only be one integer 0-255. Use stdout for other data.") + (\c -> err c 2242 "Can only exit with status 0-255. Other data should be written to stdout/stderr.")) + +returnOrExit multi invalid = (f . arguments) where f (first:second:_) = - err (getId second) 2151 - "Only one integer 0-255 can be returned. Use stdout for other data." + multi (getId first) f [value] = when (isInvalid $ literal value) $ - err (getId value) 2152 - "Can only return 0-255. Other data should be written to stdout." + invalid (getId value) f _ = return () isInvalid s = s == "" || any (not . isDigit) s || length s > 5 From 135b4aa485747a0f3ee17d153da04a28b5a0fc9d Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 26 Nov 2018 20:41:29 -0800 Subject: [PATCH 008/763] Add stack builds to distro test --- test/distrotest | 7 ++++++- test/stacktest | 27 +++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) create mode 100755 test/stacktest diff --git a/test/distrotest b/test/distrotest index 0465c3a..5024054 100755 --- a/test/distrotest +++ b/test/distrotest @@ -16,10 +16,14 @@ and is still highly experimental. Make sure you're plugged in and have screen/tmux in place, then re-run with $0 --run to continue. + +Also note that 'dist' will be deleted. EOF exit 0 } +echo "Deleting 'dist'..." +rm -rf dist log=$(mktemp) || die "Can't create temp file" date >> "$log" || die "Can't write to log" @@ -63,7 +67,8 @@ opensuse:latest zypper install -y cabal-install ghc ubuntu:18.04 apt-get update && apt-get install -y cabal-install ubuntu:17.10 apt-get update && apt-get install -y cabal-install -# Misc +# Misc Haskell including current and latest Stack build +ubuntu:18.10 set -e; apt-get update && apt-get install -y curl && curl -sSL https://get.haskellstack.org/ | sh -s - -f && cd /mnt && exec test/stacktest haskell:latest true # Known to currently fail diff --git a/test/stacktest b/test/stacktest new file mode 100755 index 0000000..dc0113f --- /dev/null +++ b/test/stacktest @@ -0,0 +1,27 @@ +#!/bin/bash +# This script builds ShellCheck through `stack` using +# various resolvers. It's run via distrotest. + +resolvers=( + nightly-"$(date -d "3 days ago" +"%Y-%m-%d")" +) + +die() { echo "$*" >&2; exit 1; } + +[ -e "ShellCheck.cabal" ] || + die "ShellCheck.cabal not in current dir" +[ -e "stack.yaml" ] || + die "stack.yaml not in current dir" +command -v stack || + die "stack is missing" + +stack setup || die "Failed to setup with default resolver" +stack build --test || die "Failed to build/test with default resolver" + +for resolver in "${resolvers[@]}" +do + stack --resolver="$resolver" setup || die "Failed to setup $resolver" + stack --resolver="$resolver" build --test || die "Failed build/test with $resolver!" +done + +echo "Success" From 1b207b3d4373b082fb4af7893421df019a1f28f3 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 26 Nov 2018 20:43:15 -0800 Subject: [PATCH 009/763] Preemptively fix possible '-- |' breakage --- src/ShellCheck/Parser.hs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index d656ed9..3330979 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -1916,8 +1916,9 @@ readNewlineList = where checkBadBreak = optional $ do pos <- getPosition - try $ lookAhead (oneOf "|&") -- |, || or && - parseProblemAt pos ErrorC 1133 "Unexpected start of line. If breaking lines, |/||/&& should be at the end of the previous one." + try $ lookAhead (oneOf "|&") -- See if the next thing could be |, || or && + parseProblemAt pos ErrorC 1133 + "Unexpected start of line. If breaking lines, |/||/&& should be at the end of the previous one." readLineBreak = optional readNewlineList prop_readSeparator1 = isWarning readScript "a &; b" From 4097bb5154110a567e4e320017fc3b6887207059 Mon Sep 17 00:00:00 2001 From: Ng Zhi An Date: Wed, 21 Nov 2018 21:36:33 -0800 Subject: [PATCH 010/763] Disable smart typography extension for markdown input Fixes #1392 --- Setup.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Setup.hs b/Setup.hs index e6064f9..a909cf6 100644 --- a/Setup.hs +++ b/Setup.hs @@ -33,4 +33,4 @@ myPreSDist _ _ = do putStrLn $ "pandoc exited with " ++ show result return emptyHookedBuildInfo where - pandoc_cmd = "pandoc -s -t man shellcheck.1.md -o shellcheck.1" + pandoc_cmd = "pandoc -s -f markdown-smart -t man shellcheck.1.md -o shellcheck.1" From b55149b22da51831eef1de250006da7963360cd1 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 2 Dec 2018 12:29:42 -0800 Subject: [PATCH 011/763] Add man page instructions (fixes #1347) --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index ba53dbb..597be37 100644 --- a/README.md +++ b/README.md @@ -172,6 +172,11 @@ Alternatively, you can download pre-compiled binaries for the latest release her or see the [storage bucket listing](https://shellcheck.storage.googleapis.com/index.html) for checksums, older versions and the latest daily builds. +Distro packages already come with a `man` page. If you are building from source, it can be installed with: + + pandoc -s -t man shellcheck.1.md -o shellcheck.1 + sudo mv shellcheck.1 /usr/share/man/man1 + ## Travis CI Travis CI has now integrated ShellCheck by default, so you don't need to manually install it. From a7a404a5a839639464f24956e0e16e2087868b6b Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 2 Dec 2018 14:49:04 -0800 Subject: [PATCH 012/763] Fill in missing bits in CHANGELOG --- CHANGELOG.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 947e456..eb00b91 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,8 +14,13 @@ - SC1117 about unknown double-quoted escape sequences has been retired ### Fixed -- SC2021 no longer triggers for equivalence classes like '[=e=]' +- SC2021 no longer triggers for equivalence classes like `[=e=]` - SC2221/SC2222 no longer mistriggers on fall-through case branches +- SC2081 about glob matches in `[ .. ]` now also triggers for `!=` +- SC2086 no longer warns about spaces in `$#` +- SC2164 no longer suggests subshells for `cd ..; cmd; cd ..` +- `read -a` is now correctly considered an array assignment +- SC2039 no longer warns about LINENO now that it's POSIX ## v0.5.0 - 2018-05-31 ### Added From 66b5f13c6ffd54ae749bfc9f6d8496de2380db0e Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 2 Dec 2018 19:07:45 -0800 Subject: [PATCH 013/763] Make wiki links fit in 80 columns --- src/ShellCheck/Formatter/TTY.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Formatter/TTY.hs b/src/ShellCheck/Formatter/TTY.hs index 64091dd..dd0e0da 100644 --- a/src/ShellCheck/Formatter/TTY.hs +++ b/src/ShellCheck/Formatter/TTY.hs @@ -94,7 +94,7 @@ outputWiki errRef = do where showErr (_, code, msg) = putStrLn $ " " ++ wikiLink ++ "SC" ++ show code ++ " -- " ++ shorten msg - limit = 40 + limit = 36 shorten msg = if length msg < limit then msg From e3f0243c0ed807880614e47014eea946f912cb20 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 2 Dec 2018 18:40:56 -0800 Subject: [PATCH 014/763] Add 'striptests' script to Cabal package --- ShellCheck.cabal | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ShellCheck.cabal b/ShellCheck.cabal index 8956ff5..5da8fac 100644 --- a/ShellCheck.cabal +++ b/ShellCheck.cabal @@ -28,6 +28,8 @@ Extra-Source-Files: shellcheck.1.md -- built with a cabal sdist hook shellcheck.1 + -- convenience script for stripping tests + striptests -- tests test/shellcheck.hs From cb57b4a74f490991e65ee8d0be1a6151a9819f91 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 2 Dec 2018 18:04:59 -0800 Subject: [PATCH 015/763] Stable version 0.6.0 This release is dedicated to Factorio. If this is how much fun it is to build factories and oppress natives, then history makes a lot of sense. --- CHANGELOG.md | 2 +- ShellCheck.cabal | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb00b91..fb9fb72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## ??? +## v0.6.0 - 2018-12-02 ### Added - Command line option --severity/-S for filtering by minimum severity - Command line option --wiki-link-count/-W for showing wiki links diff --git a/ShellCheck.cabal b/ShellCheck.cabal index 5da8fac..8a46661 100644 --- a/ShellCheck.cabal +++ b/ShellCheck.cabal @@ -1,5 +1,5 @@ Name: ShellCheck -Version: 0.5.0 +Version: 0.6.0 Synopsis: Shell script analysis tool License: GPL-3 License-file: LICENSE From 41613babd9edde4073d48d30a6c4c8cf0f97de08 Mon Sep 17 00:00:00 2001 From: Ng Zhi An Date: Sat, 13 Oct 2018 22:42:04 -0700 Subject: [PATCH 016/763] Prototype fix --- src/ShellCheck/Analytics.hs | 17 ++++++++++---- src/ShellCheck/AnalyzerLib.hs | 13 +++++++++++ src/ShellCheck/Checker.hs | 3 ++- src/ShellCheck/Formatter/TTY.hs | 39 +++++++++++++++++++++++++++++++ src/ShellCheck/Interface.hs | 41 ++++++++++++++++++++++++++++----- 5 files changed, 101 insertions(+), 12 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 3002e3e..b2e227c 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1336,7 +1336,10 @@ prop_checkBackticks1 = verify checkBackticks "echo `foo`" prop_checkBackticks2 = verifyNot checkBackticks "echo $(foo)" prop_checkBackticks3 = verifyNot checkBackticks "echo `#inlined comment` foo" checkBackticks _ (T_Backticked id list) | not (null list) = - style id 2006 "Use $(...) notation instead of legacy backticked `...`." + addComment $ + makeCommentWithFix StyleC id 2006 "Use $(...) notation instead of legacy backticked `...`." + ((replaceStart 1 "$(") ++ (replaceEnd 1 ")")) + -- style id 2006 "Use $(...) notation instead of legacy backticked `...`." checkBackticks _ _ = return () prop_checkIndirectExpansion1 = verify checkIndirectExpansion "${foo$n}" @@ -1640,8 +1643,10 @@ checkSpacefulness params t = makeComment InfoC (getId token) 2223 "This default assignment may cause DoS due to globbing. Quote it." else - makeComment InfoC (getId token) 2086 - "Double quote to prevent globbing and word splitting." + makeCommentWithFix InfoC (getId token) 2086 + "Double quote to prevent globbing and word splitting." (surroundWith "\"") + -- makeComment InfoC (getId token) 2086 + -- "Double quote to prevent globbing and word splitting." writeF _ _ name (DataString SourceExternal) = setSpaces name True >> return [] writeF _ _ name (DataString SourceInteger) = setSpaces name False >> return [] @@ -2538,7 +2543,9 @@ checkUncheckedCdPushdPopd params root = && not (isSafeDir t) && not (name t `elem` ["pushd", "popd"] && ("n" `elem` map snd (getAllFlags t))) && not (isCondition $ getPath (parentMap params) t)) $ - warn (getId t) 2164 "Use 'cd ... || exit' or 'cd ... || return' in case cd fails." + -- warn (getId t) 2164 "Use 'cd ... || exit' or 'cd ... || return' in case cd fails." + warnWithFix (getId t) 2164 "Use 'cd ... || exit' or 'cd ... || return' in case cd fails." + (replaceEnd 0 " || exit") checkElement _ = return () name t = fromMaybe "" $ getCommandName t isSafeDir t = case oversimplify t of @@ -2695,7 +2702,7 @@ checkArrayAssignmentIndices params root = T_Literal id str -> [(id,str)] _ -> [] guard $ '=' `elem` str - return $ warn id 2191 "The = here is literal. To assign by index, use ( [index]=value ) with no spaces. To keep as literal, quote it." + return $ warnWithFix id 2191 "The = here is literal. To assign by index, use ( [index]=value ) with no spaces. To keep as literal, quote it." (surroundWith "\"") in if null literalEquals && isAssociative then warn (getId t) 2190 "Elements in associative arrays need index, e.g. array=( [index]=value ) ." diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index de3498d..4bf68e7 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -150,6 +150,19 @@ err id code str = addComment $ makeComment ErrorC id code str info id code str = addComment $ makeComment InfoC id code str style id code str = addComment $ makeComment StyleC id code str +warnWithFix id code str fix = addComment $ + let comment = makeComment WarningC id code str in + comment { + tcFix = Just fix + } + +makeCommentWithFix :: Severity -> Id -> Code -> String -> Fix -> TokenComment +makeCommentWithFix severity id code str fix = + let comment = makeComment severity id code str in + comment { + tcFix = Just fix + } + makeParameters spec = let params = Parameters { rootNode = root, diff --git a/src/ShellCheck/Checker.hs b/src/ShellCheck/Checker.hs index ac58876..b9e7927 100644 --- a/src/ShellCheck/Checker.hs +++ b/src/ShellCheck/Checker.hs @@ -42,7 +42,8 @@ tokenToPosition startMap t = fromMaybe fail $ do return $ newPositionedComment { pcStartPos = fst span, pcEndPos = snd span, - pcComment = tcComment t + pcComment = tcComment t, + pcFix = tcFix t } where fail = error "Internal shellcheck error: id doesn't exist. Please report!" diff --git a/src/ShellCheck/Formatter/TTY.hs b/src/ShellCheck/Formatter/TTY.hs index dd0e0da..dc3f32e 100644 --- a/src/ShellCheck/Formatter/TTY.hs +++ b/src/ShellCheck/Formatter/TTY.hs @@ -129,8 +129,47 @@ outputForFile color sys comments = do putStrLn (color "source" line) mapM_ (\c -> putStrLn (color (severityText c) $ cuteIndent c)) x putStrLn "" + mapM_ (\c -> putStrLn "Did you mean:" >> putStrLn (fixedString c line)) x ) groups +-- need to do something smart about sorting by end index +fixedString :: PositionedComment -> String -> String +fixedString comment line = + case (pcFix comment) of + Nothing -> "" + Just rs -> + apply_replacement rs line 0 + where + apply_replacement [] s _ = s + apply_replacement ((Start n r):xs) s offset = + let start = (posColumn . pcStartPos) comment + end = start + n + z = do_replace start end s r + len_r = (fromIntegral . length) r in + apply_replacement xs z (offset + (end - start) + len_r) + apply_replacement ((End n r):xs) s offset = + -- tricky math because column is 1 based + let end = (posColumn . pcEndPos) comment + 1 + start = end - n + z = do_replace start end s r + len_r = (fromIntegral . length) r in + apply_replacement xs z (offset + (end - start) + len_r) + +-- start and end comes from pos, which is 1 based +-- do_replace 0 0 "1234" "A" -> "A1234" -- technically not valid +-- do_replace 1 1 "1234" "A" -> "A1234" +-- do_replace 1 2 "1234" "A" -> "A234" +-- do_replace 3 3 "1234" "A" -> "12A34" +-- do_replace 4 4 "1234" "A" -> "123A4" +-- do_replace 5 5 "1234" "A" -> "1234A" +do_replace start end o r = + let si = fromIntegral (start-1) + ei = fromIntegral (end-1) + (x, xs) = splitAt si o + (y, z) = splitAt (ei - si) xs + in + x ++ r ++ z + cuteIndent :: PositionedComment -> String cuteIndent comment = replicate (fromIntegral $ colNo comment - 1) ' ' ++ diff --git a/src/ShellCheck/Interface.hs b/src/ShellCheck/Interface.hs index f20874f..69d452d 100644 --- a/src/ShellCheck/Interface.hs +++ b/src/ShellCheck/Interface.hs @@ -34,9 +34,9 @@ module ShellCheck.Interface , Severity(ErrorC, WarningC, InfoC, StyleC) , Position(posFile, posLine, posColumn) , Comment(cSeverity, cCode, cMessage) - , PositionedComment(pcStartPos , pcEndPos , pcComment) + , PositionedComment(pcStartPos , pcEndPos , pcComment, pcFix) , ColorOption(ColorAuto, ColorAlways, ColorNever) - , TokenComment(tcId, tcComment) + , TokenComment(tcId, tcComment, tcFix) , emptyCheckResult , newParseResult , newAnalysisSpec @@ -49,10 +49,16 @@ module ShellCheck.Interface , emptyCheckSpec , newPositionedComment , newComment + , Fix + , Replacement(Start, End) + , surroundWith + , replaceStart + , replaceEnd ) where import ShellCheck.AST import Control.Monad.Identity +import Data.Monoid import qualified Data.Map as Map @@ -190,27 +196,50 @@ newComment = Comment { cMessage = "" } +-- only support single line for now +data Replacement = + Start Integer String + | End Integer String + deriving (Show, Eq) + +type Fix = [Replacement] + +surroundWith s = + (replaceStart 0 s) ++ (replaceEnd 0 s) + +-- replace first n chars +replaceStart n r = + [ Start n r ] + +-- replace last n chars +replaceEnd n r = + [ End n r ] + data PositionedComment = PositionedComment { pcStartPos :: Position, pcEndPos :: Position, - pcComment :: Comment + pcComment :: Comment, + pcFix :: Maybe Fix } deriving (Show, Eq) newPositionedComment :: PositionedComment newPositionedComment = PositionedComment { pcStartPos = newPosition, pcEndPos = newPosition, - pcComment = newComment + pcComment = newComment, + pcFix = Nothing } data TokenComment = TokenComment { tcId :: Id, - tcComment :: Comment + tcComment :: Comment, + tcFix :: Maybe Fix } deriving (Show, Eq) newTokenComment = TokenComment { tcId = Id 0, - tcComment = newComment + tcComment = newComment, + tcFix = Nothing } data ColorOption = From 4a87d2a3de0fa9b919dee35241a35580ab91f24a Mon Sep 17 00:00:00 2001 From: Ng Zhi An Date: Sat, 20 Oct 2018 22:09:42 -0700 Subject: [PATCH 017/763] Expose token positions in params, use that to construct fixes --- src/ShellCheck/Analytics.hs | 35 ++++++++++++++++++++++++++++----- src/ShellCheck/AnalyzerLib.hs | 6 ++++-- src/ShellCheck/Checker.hs | 18 +++++++++-------- src/ShellCheck/Formatter/TTY.hs | 22 ++++++++------------- src/ShellCheck/Interface.hs | 27 +++++++------------------ 5 files changed, 59 insertions(+), 49 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index b2e227c..d211add 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -241,6 +241,31 @@ isCondition (child:parent:rest) = T_UntilExpression id c l -> take 1 . reverse $ c _ -> [] +-- helpers to build replacements +replace_start id params n r = + let tp = tokenPositions params + (start, _) = tp Map.! id + new_end = start { + posColumn = posColumn start + n + } + in + [R start new_end r] +replace_end id params n r = + -- because of the way we count columns 1-based + -- we have to offset end columns by 1 + let tp = tokenPositions params + (_, end) = tp Map.! id + new_start = end { + posColumn = posColumn end - n + 1 + } + new_end = end { + posColumn = posColumn end + 1 + } + in + [R new_start new_end r] +surround_with id params s = + (replace_start id params 0 s) ++ (replace_end id params 0 s) + prop_checkEchoWc3 = verify checkEchoWc "n=$(echo $foo | wc -c)" checkEchoWc _ (T_Pipeline id _ [a, b]) = when (acmd == ["echo", "${VAR}"]) $ @@ -1335,10 +1360,10 @@ checkPS1Assignments _ _ = return () prop_checkBackticks1 = verify checkBackticks "echo `foo`" prop_checkBackticks2 = verifyNot checkBackticks "echo $(foo)" prop_checkBackticks3 = verifyNot checkBackticks "echo `#inlined comment` foo" -checkBackticks _ (T_Backticked id list) | not (null list) = +checkBackticks params (T_Backticked id list) | not (null list) = addComment $ makeCommentWithFix StyleC id 2006 "Use $(...) notation instead of legacy backticked `...`." - ((replaceStart 1 "$(") ++ (replaceEnd 1 ")")) + ((replace_start id params 1 "$(") ++ (replace_end id params 1 ")")) -- style id 2006 "Use $(...) notation instead of legacy backticked `...`." checkBackticks _ _ = return () @@ -1644,7 +1669,7 @@ checkSpacefulness params t = "This default assignment may cause DoS due to globbing. Quote it." else makeCommentWithFix InfoC (getId token) 2086 - "Double quote to prevent globbing and word splitting." (surroundWith "\"") + "Double quote to prevent globbing and word splitting." (surround_with (getId token) params "\"") -- makeComment InfoC (getId token) 2086 -- "Double quote to prevent globbing and word splitting." @@ -2545,7 +2570,7 @@ checkUncheckedCdPushdPopd params root = && not (isCondition $ getPath (parentMap params) t)) $ -- warn (getId t) 2164 "Use 'cd ... || exit' or 'cd ... || return' in case cd fails." warnWithFix (getId t) 2164 "Use 'cd ... || exit' or 'cd ... || return' in case cd fails." - (replaceEnd 0 " || exit") + (replace_end (getId t) params 0 " || exit") checkElement _ = return () name t = fromMaybe "" $ getCommandName t isSafeDir t = case oversimplify t of @@ -2702,7 +2727,7 @@ checkArrayAssignmentIndices params root = T_Literal id str -> [(id,str)] _ -> [] guard $ '=' `elem` str - return $ warnWithFix id 2191 "The = here is literal. To assign by index, use ( [index]=value ) with no spaces. To keep as literal, quote it." (surroundWith "\"") + return $ warnWithFix id 2191 "The = here is literal. To assign by index, use ( [index]=value ) with no spaces. To keep as literal, quote it." (surround_with id params "\"") in if null literalEquals && isAssociative then warn (getId t) 2190 "Elements in associative arrays need index, e.g. array=( [index]=value ) ." diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 4bf68e7..1639ff6 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -81,7 +81,8 @@ data Parameters = Parameters { parentMap :: Map.Map Id Token, -- A map from Id to parent Token shellType :: Shell, -- The shell type, such as Bash or Ksh shellTypeSpecified :: Bool, -- True if shell type was forced via flags - rootNode :: Token -- The root node of the AST + rootNode :: Token, -- The root node of the AST + tokenPositions :: Map.Map Id (Position, Position) -- map from token id to start and end position } -- TODO: Cache results of common AST ops here @@ -177,7 +178,8 @@ makeParameters spec = shellTypeSpecified = isJust $ asShellType spec, parentMap = getParentTree root, - variableFlow = getVariableFlow params root + variableFlow = getVariableFlow params root, + tokenPositions = asTokenPositions spec } in params where root = asScript spec diff --git a/src/ShellCheck/Checker.hs b/src/ShellCheck/Checker.hs index b9e7927..7ac9c91 100644 --- a/src/ShellCheck/Checker.hs +++ b/src/ShellCheck/Checker.hs @@ -64,11 +64,20 @@ checkScript sys spec = do psShellTypeOverride = csShellTypeOverride spec } let parseMessages = prComments result + let tokenPositions = prTokenPositions result + let analysisSpec root = + as { + asScript = root, + asShellType = csShellTypeOverride spec, + asCheckSourced = csCheckSourced spec, + asExecutionMode = Executed, + asTokenPositions = tokenPositions + } where as = newAnalysisSpec root let analysisMessages = fromMaybe [] $ (arComments . analyzeScript . analysisSpec) <$> prRoot result - let translator = tokenToPosition (prTokenPositions result) + let translator = tokenToPosition tokenPositions return . nub . sortMessages . filter shouldInclude $ (parseMessages ++ map translator analysisMessages) @@ -91,13 +100,6 @@ checkScript sys spec = do cMessage comment) getPosition = pcStartPos - analysisSpec root = - as { - asScript = root, - asShellType = csShellTypeOverride spec, - asCheckSourced = csCheckSourced spec, - asExecutionMode = Executed - } where as = newAnalysisSpec root getErrors sys spec = sort . map getCode . crComments $ diff --git a/src/ShellCheck/Formatter/TTY.hs b/src/ShellCheck/Formatter/TTY.hs index dc3f32e..a1c98b1 100644 --- a/src/ShellCheck/Formatter/TTY.hs +++ b/src/ShellCheck/Formatter/TTY.hs @@ -118,8 +118,8 @@ outputForFile color sys comments = do let fileLines = lines contents let lineCount = fromIntegral $ length fileLines let groups = groupWith lineNo comments - mapM_ (\x -> do - let lineNum = lineNo (head x) + mapM_ (\commentsForLine -> do + let lineNum = lineNo (head commentsForLine) let line = if lineNum < 1 || lineNum > lineCount then "" else fileLines !! fromIntegral (lineNum - 1) @@ -127,9 +127,10 @@ outputForFile color sys comments = do putStrLn $ color "message" $ "In " ++ fileName ++" line " ++ show lineNum ++ ":" putStrLn (color "source" line) - mapM_ (\c -> putStrLn (color (severityText c) $ cuteIndent c)) x + mapM_ (\c -> putStrLn (color (severityText c) $ cuteIndent c)) commentsForLine putStrLn "" - mapM_ (\c -> putStrLn "Did you mean:" >> putStrLn (fixedString c line)) x + -- in the spirit of error prone + mapM_ (\c -> putStrLn "Did you mean:" >> putStrLn (fixedString c line)) commentsForLine ) groups -- need to do something smart about sorting by end index @@ -141,16 +142,9 @@ fixedString comment line = apply_replacement rs line 0 where apply_replacement [] s _ = s - apply_replacement ((Start n r):xs) s offset = - let start = (posColumn . pcStartPos) comment - end = start + n - z = do_replace start end s r - len_r = (fromIntegral . length) r in - apply_replacement xs z (offset + (end - start) + len_r) - apply_replacement ((End n r):xs) s offset = - -- tricky math because column is 1 based - let end = (posColumn . pcEndPos) comment + 1 - start = end - n + apply_replacement ((R startp endp r):xs) s offset = + let start = posColumn startp + end = posColumn endp z = do_replace start end s r len_r = (fromIntegral . length) r in apply_replacement xs z (offset + (end - start) + len_r) diff --git a/src/ShellCheck/Interface.hs b/src/ShellCheck/Interface.hs index 69d452d..ef9dd44 100644 --- a/src/ShellCheck/Interface.hs +++ b/src/ShellCheck/Interface.hs @@ -24,7 +24,7 @@ module ShellCheck.Interface , CheckResult(crFilename, crComments) , ParseSpec(psFilename, psScript, psCheckSourced, psShellTypeOverride) , ParseResult(prComments, prTokenPositions, prRoot) - , AnalysisSpec(asScript, asShellType, asExecutionMode, asCheckSourced) + , AnalysisSpec(asScript, asShellType, asExecutionMode, asCheckSourced, asTokenPositions) , AnalysisResult(arComments) , FormatterOptions(foColorOption, foWikiLinkCount) , Shell(Ksh, Sh, Bash, Dash) @@ -50,10 +50,7 @@ module ShellCheck.Interface , newPositionedComment , newComment , Fix - , Replacement(Start, End) - , surroundWith - , replaceStart - , replaceEnd + , Replacement(R) ) where import ShellCheck.AST @@ -132,14 +129,16 @@ data AnalysisSpec = AnalysisSpec { asScript :: Token, asShellType :: Maybe Shell, asExecutionMode :: ExecutionMode, - asCheckSourced :: Bool + asCheckSourced :: Bool, + asTokenPositions :: Map.Map Id (Position, Position) } newAnalysisSpec token = AnalysisSpec { asScript = token, asShellType = Nothing, asExecutionMode = Executed, - asCheckSourced = False + asCheckSourced = False, + asTokenPositions = Map.empty } newtype AnalysisResult = AnalysisResult { @@ -198,23 +197,11 @@ newComment = Comment { -- only support single line for now data Replacement = - Start Integer String - | End Integer String + R Position Position String deriving (Show, Eq) type Fix = [Replacement] -surroundWith s = - (replaceStart 0 s) ++ (replaceEnd 0 s) - --- replace first n chars -replaceStart n r = - [ Start n r ] - --- replace last n chars -replaceEnd n r = - [ End n r ] - data PositionedComment = PositionedComment { pcStartPos :: Position, pcEndPos :: Position, From 5ed89d22411b407b94cab060e6395a235118bc47 Mon Sep 17 00:00:00 2001 From: Ng Zhi An Date: Sat, 20 Oct 2018 23:20:30 -0700 Subject: [PATCH 018/763] Change definition of Replacement, add ToJSON instance for it --- src/ShellCheck/Analytics.hs | 18 +++++++++++++----- src/ShellCheck/Formatter/JSON.hs | 18 ++++++++++++++++-- src/ShellCheck/Formatter/TTY.hs | 11 ++++++----- src/ShellCheck/Interface.hs | 17 +++++++++++++---- 4 files changed, 48 insertions(+), 16 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index d211add..a034031 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -249,7 +249,11 @@ replace_start id params n r = posColumn = posColumn start + n } in - [R start new_end r] + newReplacement { + repStartPos = start, + repEndPos = new_end, + repString = r + } replace_end id params n r = -- because of the way we count columns 1-based -- we have to offset end columns by 1 @@ -262,9 +266,13 @@ replace_end id params n r = posColumn = posColumn end + 1 } in - [R new_start new_end r] + newReplacement { + repStartPos = new_start, + repEndPos = new_end, + repString = r + } surround_with id params s = - (replace_start id params 0 s) ++ (replace_end id params 0 s) + [replace_start id params 0 s, replace_end id params 0 s] prop_checkEchoWc3 = verify checkEchoWc "n=$(echo $foo | wc -c)" checkEchoWc _ (T_Pipeline id _ [a, b]) = @@ -1363,7 +1371,7 @@ prop_checkBackticks3 = verifyNot checkBackticks "echo `#inlined comment` foo" checkBackticks params (T_Backticked id list) | not (null list) = addComment $ makeCommentWithFix StyleC id 2006 "Use $(...) notation instead of legacy backticked `...`." - ((replace_start id params 1 "$(") ++ (replace_end id params 1 ")")) + [(replace_start id params 1 "$("), (replace_end id params 1 ")")] -- style id 2006 "Use $(...) notation instead of legacy backticked `...`." checkBackticks _ _ = return () @@ -2570,7 +2578,7 @@ checkUncheckedCdPushdPopd params root = && not (isCondition $ getPath (parentMap params) t)) $ -- warn (getId t) 2164 "Use 'cd ... || exit' or 'cd ... || return' in case cd fails." warnWithFix (getId t) 2164 "Use 'cd ... || exit' or 'cd ... || return' in case cd fails." - (replace_end (getId t) params 0 " || exit") + [replace_end (getId t) params 0 " || exit"] checkElement _ = return () name t = fromMaybe "" $ getCommandName t isSafeDir t = case oversimplify t of diff --git a/src/ShellCheck/Formatter/JSON.hs b/src/ShellCheck/Formatter/JSON.hs index aac4d20..072af7e 100644 --- a/src/ShellCheck/Formatter/JSON.hs +++ b/src/ShellCheck/Formatter/JSON.hs @@ -39,6 +39,19 @@ format = do footer = finish ref } +instance ToJSON Replacement where + toJSON replacement = + let start = repStartPos replacement + end = repEndPos replacement + str = repString replacement in + object [ + "line" .= posLine start, + "endLine" .= posLine end, + "column" .= posColumn start, + "endColumn" .= posColumn end, + "replaceWith" .= str + ] + instance ToJSON (PositionedComment) where toJSON comment = let start = pcStartPos comment @@ -52,7 +65,8 @@ instance ToJSON (PositionedComment) where "endColumn" .= posColumn end, "level" .= severityText comment, "code" .= cCode c, - "message" .= cMessage c + "message" .= cMessage c, + "fix" .= pcFix comment ] toEncoding comment = @@ -68,6 +82,7 @@ instance ToJSON (PositionedComment) where <> "level" .= severityText comment <> "code" .= cCode c <> "message" .= cMessage c + <> "replaceWith" .= pcFix comment ) outputError file msg = hPutStrLn stderr $ file ++ ": " ++ msg @@ -77,4 +92,3 @@ collectResult ref result _ = finish ref = do list <- readIORef ref BL.putStrLn $ encode list - diff --git a/src/ShellCheck/Formatter/TTY.hs b/src/ShellCheck/Formatter/TTY.hs index a1c98b1..224d6fe 100644 --- a/src/ShellCheck/Formatter/TTY.hs +++ b/src/ShellCheck/Formatter/TTY.hs @@ -142,11 +142,12 @@ fixedString comment line = apply_replacement rs line 0 where apply_replacement [] s _ = s - apply_replacement ((R startp endp r):xs) s offset = - let start = posColumn startp - end = posColumn endp - z = do_replace start end s r - len_r = (fromIntegral . length) r in + apply_replacement (rep:xs) s offset = + let replacementString = repString rep + start = (posColumn . repStartPos) rep + end = (posColumn . repEndPos) rep + z = do_replace start end s replacementString + len_r = (fromIntegral . length) replacementString in apply_replacement xs z (offset + (end - start) + len_r) -- start and end comes from pos, which is 1 based diff --git a/src/ShellCheck/Interface.hs b/src/ShellCheck/Interface.hs index ef9dd44..a429a88 100644 --- a/src/ShellCheck/Interface.hs +++ b/src/ShellCheck/Interface.hs @@ -50,7 +50,8 @@ module ShellCheck.Interface , newPositionedComment , newComment , Fix - , Replacement(R) + , Replacement(repStartPos, repEndPos, repString) + , newReplacement ) where import ShellCheck.AST @@ -196,9 +197,17 @@ newComment = Comment { } -- only support single line for now -data Replacement = - R Position Position String - deriving (Show, Eq) +data Replacement = Replacement { + repStartPos :: Position, + repEndPos :: Position, + repString :: String +} deriving (Show, Eq) + +newReplacement = Replacement { + repStartPos = newPosition, + repEndPos = newPosition, + repString = "" +} type Fix = [Replacement] From a8376a09a90cab4881ef43905dbcca461a302d77 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 22 Oct 2018 18:41:36 -0700 Subject: [PATCH 019/763] Minor renaming and output fixes --- src/ShellCheck/Analytics.hs | 18 ++++++------ src/ShellCheck/Formatter/JSON.hs | 9 ++++-- src/ShellCheck/Formatter/TTY.hs | 47 ++++++++++++++++++++++---------- src/ShellCheck/Interface.hs | 11 ++++++-- 4 files changed, 57 insertions(+), 28 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index a034031..bbd7453 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -242,7 +242,7 @@ isCondition (child:parent:rest) = _ -> [] -- helpers to build replacements -replace_start id params n r = +replaceStart id params n r = let tp = tokenPositions params (start, _) = tp Map.! id new_end = start { @@ -254,7 +254,7 @@ replace_start id params n r = repEndPos = new_end, repString = r } -replace_end id params n r = +replaceEnd id params n r = -- because of the way we count columns 1-based -- we have to offset end columns by 1 let tp = tokenPositions params @@ -271,8 +271,8 @@ replace_end id params n r = repEndPos = new_end, repString = r } -surround_with id params s = - [replace_start id params 0 s, replace_end id params 0 s] +surroundWidth id params s = fixWith [replaceStart id params 0 s, replaceEnd id params 0 s] +fixWith fixes = newFix { fixReplacements = fixes } prop_checkEchoWc3 = verify checkEchoWc "n=$(echo $foo | wc -c)" checkEchoWc _ (T_Pipeline id _ [a, b]) = @@ -1371,8 +1371,7 @@ prop_checkBackticks3 = verifyNot checkBackticks "echo `#inlined comment` foo" checkBackticks params (T_Backticked id list) | not (null list) = addComment $ makeCommentWithFix StyleC id 2006 "Use $(...) notation instead of legacy backticked `...`." - [(replace_start id params 1 "$("), (replace_end id params 1 ")")] - -- style id 2006 "Use $(...) notation instead of legacy backticked `...`." + (fixWith [replaceStart id params 1 "$(", replaceEnd id params 1 ")"]) checkBackticks _ _ = return () prop_checkIndirectExpansion1 = verify checkIndirectExpansion "${foo$n}" @@ -1677,7 +1676,7 @@ checkSpacefulness params t = "This default assignment may cause DoS due to globbing. Quote it." else makeCommentWithFix InfoC (getId token) 2086 - "Double quote to prevent globbing and word splitting." (surround_with (getId token) params "\"") + "Double quote to prevent globbing and word splitting." (surroundWidth (getId token) params "\"") -- makeComment InfoC (getId token) 2086 -- "Double quote to prevent globbing and word splitting." @@ -2576,9 +2575,8 @@ checkUncheckedCdPushdPopd params root = && not (isSafeDir t) && not (name t `elem` ["pushd", "popd"] && ("n" `elem` map snd (getAllFlags t))) && not (isCondition $ getPath (parentMap params) t)) $ - -- warn (getId t) 2164 "Use 'cd ... || exit' or 'cd ... || return' in case cd fails." warnWithFix (getId t) 2164 "Use 'cd ... || exit' or 'cd ... || return' in case cd fails." - [replace_end (getId t) params 0 " || exit"] + (fixWith [replaceEnd (getId t) params 0 " || exit"]) checkElement _ = return () name t = fromMaybe "" $ getCommandName t isSafeDir t = case oversimplify t of @@ -2735,7 +2733,7 @@ checkArrayAssignmentIndices params root = T_Literal id str -> [(id,str)] _ -> [] guard $ '=' `elem` str - return $ warnWithFix id 2191 "The = here is literal. To assign by index, use ( [index]=value ) with no spaces. To keep as literal, quote it." (surround_with id params "\"") + return $ warnWithFix id 2191 "The = here is literal. To assign by index, use ( [index]=value ) with no spaces. To keep as literal, quote it." (surroundWidth id params "\"") in if null literalEquals && isAssociative then warn (getId t) 2190 "Elements in associative arrays need index, e.g. array=( [index]=value ) ." diff --git a/src/ShellCheck/Formatter/JSON.hs b/src/ShellCheck/Formatter/JSON.hs index 072af7e..9aec751 100644 --- a/src/ShellCheck/Formatter/JSON.hs +++ b/src/ShellCheck/Formatter/JSON.hs @@ -52,7 +52,7 @@ instance ToJSON Replacement where "replaceWith" .= str ] -instance ToJSON (PositionedComment) where +instance ToJSON PositionedComment where toJSON comment = let start = pcStartPos comment end = pcEndPos comment @@ -82,9 +82,14 @@ instance ToJSON (PositionedComment) where <> "level" .= severityText comment <> "code" .= cCode c <> "message" .= cMessage c - <> "replaceWith" .= pcFix comment + <> "fix" .= pcFix comment ) +instance ToJSON Fix where + toJSON fix = object [ + "replacements" .= fixReplacements fix + ] + outputError file msg = hPutStrLn stderr $ file ++ ": " ++ msg collectResult ref result _ = modifyIORef ref (\x -> crComments result ++ x) diff --git a/src/ShellCheck/Formatter/TTY.hs b/src/ShellCheck/Formatter/TTY.hs index 224d6fe..a5d7490 100644 --- a/src/ShellCheck/Formatter/TTY.hs +++ b/src/ShellCheck/Formatter/TTY.hs @@ -25,6 +25,7 @@ import ShellCheck.Formatter.Format import Control.Monad import Data.IORef import Data.List +import Data.Maybe import GHC.Exts import System.IO import System.Info @@ -129,35 +130,53 @@ outputForFile color sys comments = do putStrLn (color "source" line) mapM_ (\c -> putStrLn (color (severityText c) $ cuteIndent c)) commentsForLine putStrLn "" - -- in the spirit of error prone - mapM_ (\c -> putStrLn "Did you mean:" >> putStrLn (fixedString c line)) commentsForLine + -- FIXME: Enable when reasonably stable + -- showFixedString color comments lineNum line ) groups +hasApplicableFix lineNum comment = fromMaybe False $ do + replacements <- fixReplacements <$> pcFix comment + guard $ all (\c -> onSameLine (repStartPos c) && onSameLine (repEndPos c)) replacements + return True + where + onSameLine pos = posLine pos == lineNum + +-- FIXME: Work correctly with multiple replacements +showFixedString color comments lineNum line = + case filter (hasApplicableFix lineNum) comments of + (first:_) -> do + -- in the spirit of error prone + putStrLn $ color "message" "Did you mean: " + putStrLn $ fixedString first line + putStrLn "" + _ -> return () + -- need to do something smart about sorting by end index fixedString :: PositionedComment -> String -> String fixedString comment line = case (pcFix comment) of Nothing -> "" Just rs -> - apply_replacement rs line 0 + applyReplacement (fixReplacements rs) line 0 where - apply_replacement [] s _ = s - apply_replacement (rep:xs) s offset = + applyReplacement [] s _ = s + applyReplacement (rep:xs) s offset = let replacementString = repString rep start = (posColumn . repStartPos) rep end = (posColumn . repEndPos) rep - z = do_replace start end s replacementString + z = doReplace start end s replacementString len_r = (fromIntegral . length) replacementString in - apply_replacement xs z (offset + (end - start) + len_r) + applyReplacement xs z (offset + (end - start) + len_r) +-- FIXME: Work correctly with tabs -- start and end comes from pos, which is 1 based --- do_replace 0 0 "1234" "A" -> "A1234" -- technically not valid --- do_replace 1 1 "1234" "A" -> "A1234" --- do_replace 1 2 "1234" "A" -> "A234" --- do_replace 3 3 "1234" "A" -> "12A34" --- do_replace 4 4 "1234" "A" -> "123A4" --- do_replace 5 5 "1234" "A" -> "1234A" -do_replace start end o r = +-- doReplace 0 0 "1234" "A" -> "A1234" -- technically not valid +-- doReplace 1 1 "1234" "A" -> "A1234" +-- doReplace 1 2 "1234" "A" -> "A234" +-- doReplace 3 3 "1234" "A" -> "12A34" +-- doReplace 4 4 "1234" "A" -> "123A4" +-- doReplace 5 5 "1234" "A" -> "1234A" +doReplace start end o r = let si = fromIntegral (start-1) ei = fromIntegral (end-1) (x, xs) = splitAt si o diff --git a/src/ShellCheck/Interface.hs b/src/ShellCheck/Interface.hs index a429a88..4a7214b 100644 --- a/src/ShellCheck/Interface.hs +++ b/src/ShellCheck/Interface.hs @@ -49,7 +49,8 @@ module ShellCheck.Interface , emptyCheckSpec , newPositionedComment , newComment - , Fix + , Fix(fixReplacements) + , newFix , Replacement(repStartPos, repEndPos, repString) , newReplacement ) where @@ -209,7 +210,13 @@ newReplacement = Replacement { repString = "" } -type Fix = [Replacement] +data Fix = Fix { + fixReplacements :: [Replacement] +} deriving (Show, Eq) + +newFix = Fix { + fixReplacements = [] +} data PositionedComment = PositionedComment { pcStartPos :: Position, From bcd13614ebf026774fb5c2e9ae47236648704aac Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 22 Oct 2018 19:39:24 -0700 Subject: [PATCH 020/763] Improve Fix memory usage --- ShellCheck.cabal | 3 +++ src/ShellCheck/AST.hs | 5 +++- src/ShellCheck/AnalyzerLib.hs | 49 +++++++++++++++++++---------------- src/ShellCheck/Interface.hs | 19 +++++++++----- 4 files changed, 45 insertions(+), 31 deletions(-) diff --git a/ShellCheck.cabal b/ShellCheck.cabal index 8a46661..721da3f 100644 --- a/ShellCheck.cabal +++ b/ShellCheck.cabal @@ -55,6 +55,7 @@ library base > 4.6.0.1 && < 5, bytestring, containers >= 0.5, + deepseq >= 1.4.0.0, directory, mtl >= 2.2.1, parsec, @@ -91,6 +92,7 @@ executable shellcheck aeson, base >= 4 && < 5, bytestring, + deepseq >= 1.4.0.0, ShellCheck, containers, directory, @@ -106,6 +108,7 @@ test-suite test-shellcheck aeson, base >= 4 && < 5, bytestring, + deepseq >= 1.4.0.0, ShellCheck, containers, directory, diff --git a/src/ShellCheck/AST.hs b/src/ShellCheck/AST.hs index cd96165..8a6d7b2 100644 --- a/src/ShellCheck/AST.hs +++ b/src/ShellCheck/AST.hs @@ -17,14 +17,17 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . -} +{-# LANGUAGE DeriveGeneric, DeriveAnyClass #-} module ShellCheck.AST where +import GHC.Generics (Generic) import Control.Monad.Identity +import Control.DeepSeq import Text.Parsec import qualified ShellCheck.Regex as Re import Prelude hiding (id) -newtype Id = Id Int deriving (Show, Eq, Ord) +newtype Id = Id Int deriving (Show, Eq, Ord, Generic, NFData) data Quoted = Quoted | Unquoted deriving (Show, Eq) data Dashed = Dashed | Undashed deriving (Show, Eq) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 1639ff6..9b7892d 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -20,26 +20,28 @@ {-# LANGUAGE FlexibleContexts #-} {-# LANGUAGE TemplateHaskell #-} module ShellCheck.AnalyzerLib where -import ShellCheck.AST -import ShellCheck.ASTLib -import ShellCheck.Data -import ShellCheck.Interface -import ShellCheck.Parser -import ShellCheck.Regex -import Control.Arrow (first) -import Control.Monad.Identity -import Control.Monad.RWS -import Control.Monad.State -import Control.Monad.Writer -import Data.Char -import Data.List -import qualified Data.Map as Map -import Data.Maybe -import Data.Semigroup +import ShellCheck.AST +import ShellCheck.ASTLib +import ShellCheck.Data +import ShellCheck.Interface +import ShellCheck.Parser +import ShellCheck.Regex -import Test.QuickCheck.All (forAllProperties) -import Test.QuickCheck.Test (maxSuccess, quickCheckWithResult, stdArgs) +import Control.Arrow (first) +import Control.DeepSeq +import Control.Monad.Identity +import Control.Monad.RWS +import Control.Monad.State +import Control.Monad.Writer +import Data.Char +import Data.List +import Data.Maybe +import Data.Semigroup +import qualified Data.Map as Map + +import Test.QuickCheck.All (forAllProperties) +import Test.QuickCheck.Test (maxSuccess, quickCheckWithResult, stdArgs) type Analysis = AnalyzerM () type AnalyzerM a = RWS Parameters [TokenComment] Cache a @@ -143,7 +145,7 @@ makeComment severity id code note = } } -addComment note = tell [note] +addComment note = note `deepseq` tell [note] warn :: MonadWriter [TokenComment] m => Id -> Code -> String -> m () warn id code str = addComment $ makeComment WarningC id code str @@ -159,10 +161,11 @@ warnWithFix id code str fix = addComment $ makeCommentWithFix :: Severity -> Id -> Code -> String -> Fix -> TokenComment makeCommentWithFix severity id code str fix = - let comment = makeComment severity id code str in - comment { - tcFix = Just fix - } + let comment = makeComment severity id code str + withFix = comment { + tcFix = Just fix + } + in withFix `deepseq` withFix makeParameters spec = let params = Parameters { diff --git a/src/ShellCheck/Interface.hs b/src/ShellCheck/Interface.hs index 4a7214b..092b9e8 100644 --- a/src/ShellCheck/Interface.hs +++ b/src/ShellCheck/Interface.hs @@ -17,6 +17,7 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . -} +{-# LANGUAGE DeriveGeneric, DeriveAnyClass #-} module ShellCheck.Interface ( SystemInterface(..) @@ -56,8 +57,11 @@ module ShellCheck.Interface ) where import ShellCheck.AST + +import Control.DeepSeq import Control.Monad.Identity import Data.Monoid +import GHC.Generics (Generic) import qualified Data.Map as Map @@ -170,12 +174,13 @@ data ExecutionMode = Executed | Sourced deriving (Show, Eq) type ErrorMessage = String type Code = Integer -data Severity = ErrorC | WarningC | InfoC | StyleC deriving (Show, Eq, Ord) +data Severity = ErrorC | WarningC | InfoC | StyleC + deriving (Show, Eq, Ord, Generic, NFData) data Position = Position { posFile :: String, -- Filename posLine :: Integer, -- 1 based source line posColumn :: Integer -- 1 based source column, where tabs are 8 -} deriving (Show, Eq) +} deriving (Show, Eq, Generic, NFData) newPosition :: Position newPosition = Position { @@ -188,7 +193,7 @@ data Comment = Comment { cSeverity :: Severity, cCode :: Code, cMessage :: String -} deriving (Show, Eq) +} deriving (Show, Eq, Generic, NFData) newComment :: Comment newComment = Comment { @@ -202,7 +207,7 @@ data Replacement = Replacement { repStartPos :: Position, repEndPos :: Position, repString :: String -} deriving (Show, Eq) +} deriving (Show, Eq, Generic, NFData) newReplacement = Replacement { repStartPos = newPosition, @@ -212,7 +217,7 @@ newReplacement = Replacement { data Fix = Fix { fixReplacements :: [Replacement] -} deriving (Show, Eq) +} deriving (Show, Eq, Generic, NFData) newFix = Fix { fixReplacements = [] @@ -223,7 +228,7 @@ data PositionedComment = PositionedComment { pcEndPos :: Position, pcComment :: Comment, pcFix :: Maybe Fix -} deriving (Show, Eq) +} deriving (Show, Eq, Generic, NFData) newPositionedComment :: PositionedComment newPositionedComment = PositionedComment { @@ -237,7 +242,7 @@ data TokenComment = TokenComment { tcId :: Id, tcComment :: Comment, tcFix :: Maybe Fix -} deriving (Show, Eq) +} deriving (Show, Eq, Generic, NFData) newTokenComment = TokenComment { tcId = Id 0, From eb588f62f6385df447af24338638983d82ab1d7b Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 9 Dec 2018 14:59:36 -0800 Subject: [PATCH 021/763] Enable autofix support. It's still preliminary. --- src/ShellCheck/Formatter/TTY.hs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/ShellCheck/Formatter/TTY.hs b/src/ShellCheck/Formatter/TTY.hs index a5d7490..2d6c010 100644 --- a/src/ShellCheck/Formatter/TTY.hs +++ b/src/ShellCheck/Formatter/TTY.hs @@ -130,8 +130,7 @@ outputForFile color sys comments = do putStrLn (color "source" line) mapM_ (\c -> putStrLn (color (severityText c) $ cuteIndent c)) commentsForLine putStrLn "" - -- FIXME: Enable when reasonably stable - -- showFixedString color comments lineNum line + showFixedString color comments lineNum line ) groups hasApplicableFix lineNum comment = fromMaybe False $ do From 3cba76dc7da3037148b44ea0ee1f5fea2f18e79d Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 9 Dec 2018 14:38:15 -0800 Subject: [PATCH 022/763] Update CHANGELOG with new release and autofix merge --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fb9fb72..6c239e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## Since previous release +### Added +- Preliminary support for fix suggestions + ## v0.6.0 - 2018-12-02 ### Added - Command line option --severity/-S for filtering by minimum severity From b47e083ee340b1875fc3fce9921ca1c447918e4e Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 16 Dec 2018 10:15:03 -0800 Subject: [PATCH 023/763] Fix 'does not support multiple targets at once' error --- quicktest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quicktest b/quicktest index 42a7ba6..4f0702d 100755 --- a/quicktest +++ b/quicktest @@ -11,7 +11,7 @@ ,ShellCheck.Checks.Commands.runTests ,ShellCheck.Checks.ShellSupport.runTests ,ShellCheck.AnalyzerLib.runTests - ]' | tr -d '\n' | cabal repl 2>&1 | tee /dev/stderr) + ]' | tr -d '\n' | cabal repl ShellCheck 2>&1 | tee /dev/stderr) if [[ $var == *$'\nTrue'* ]] then exit 0 From 5b3f17c29d903cb5a20f7ed58c732b69e51cf68c Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 16 Dec 2018 13:05:19 -0800 Subject: [PATCH 024/763] Allow tests to access token positions for fixes --- src/ShellCheck/Analytics.hs | 5 +++-- src/ShellCheck/AnalyzerLib.hs | 36 ++++++++++++++++++----------------- 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index bbd7453..7754ee6 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -199,8 +199,9 @@ checkUnqualifiedCommand _ _ _ = return () checkNode f = producesComments (runNodeAnalysis f) producesComments :: (Parameters -> Token -> [TokenComment]) -> String -> Maybe Bool producesComments f s = do - root <- pScript s - return . not . null $ runList (defaultSpec root) [f] + let pr = pScript s + prRoot pr + return . not . null $ runList (defaultSpec pr) [f] -- Copied from https://wiki.haskell.org/Edit_distance dist :: Eq a => [a] -> [a] -> Int diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 9b7892d..306c139 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -85,7 +85,7 @@ data Parameters = Parameters { shellTypeSpecified :: Bool, -- True if shell type was forced via flags rootNode :: Token, -- The root node of the AST tokenPositions :: Map.Map Id (Position, Position) -- map from token id to start and end position - } + } deriving (Show) -- TODO: Cache results of common AST ops here data Cache = Cache {} @@ -112,11 +112,12 @@ data DataSource = data VariableState = Dead Token String | Alive deriving (Show) -defaultSpec root = spec { +defaultSpec pr = spec { asShellType = Nothing, asCheckSourced = False, - asExecutionMode = Executed -} where spec = newAnalysisSpec root + asExecutionMode = Executed, + asTokenPositions = prTokenPositions pr +} where spec = newAnalysisSpec (fromJust $ prRoot pr) pScript s = let @@ -124,13 +125,14 @@ pScript s = psFilename = "script", psScript = s } - in prRoot . runIdentity $ parseScript (mockedSystemInterface []) pSpec + in runIdentity $ parseScript (mockedSystemInterface []) pSpec -- For testing. If parsed, returns whether there are any comments producesComments :: Checker -> String -> Maybe Bool producesComments c s = do - root <- pScript s - let spec = defaultSpec root + let pr = pScript s + prRoot pr + let spec = defaultSpec pr let params = makeParameters spec return . not . null $ runChecker params c @@ -214,16 +216,16 @@ containsLastpipe root = _ -> False -prop_determineShell0 = determineShell (fromJust $ pScript "#!/bin/sh") == Sh -prop_determineShell1 = determineShell (fromJust $ pScript "#!/usr/bin/env ksh") == Ksh -prop_determineShell2 = determineShell (fromJust $ pScript "") == Bash -prop_determineShell3 = determineShell (fromJust $ pScript "#!/bin/sh -e") == Sh -prop_determineShell4 = determineShell (fromJust $ pScript - "#!/bin/ksh\n#shellcheck shell=sh\nfoo") == Sh -prop_determineShell5 = determineShell (fromJust $ pScript - "#shellcheck shell=sh\nfoo") == Sh -prop_determineShell6 = determineShell (fromJust $ pScript "#! /bin/sh") == Sh -prop_determineShell7 = determineShell (fromJust $ pScript "#! /bin/ash") == Dash +prop_determineShell0 = determineShellTest "#!/bin/sh" == Sh +prop_determineShell1 = determineShellTest "#!/usr/bin/env ksh" == Ksh +prop_determineShell2 = determineShellTest "" == Bash +prop_determineShell3 = determineShellTest "#!/bin/sh -e" == Sh +prop_determineShell4 = determineShellTest "#!/bin/ksh\n#shellcheck shell=sh\nfoo" == Sh +prop_determineShell5 = determineShellTest "#shellcheck shell=sh\nfoo" == Sh +prop_determineShell6 = determineShellTest "#! /bin/sh" == Sh +prop_determineShell7 = determineShellTest "#! /bin/ash" == Dash + +determineShellTest = determineShell . fromJust . prRoot . pScript determineShell t = fromMaybe Bash $ do shellString <- foldl mplus Nothing $ getCandidates t shellForExecutable shellString From 138080bdc7442458f45bd2726d9e8cb4b68cf24f Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 16 Dec 2018 14:42:19 -0800 Subject: [PATCH 025/763] Fix infinite loop on annotations for SC2188 (fixes #1413) --- src/ShellCheck/Analytics.hs | 2 +- src/ShellCheck/Checker.hs | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 7754ee6..a164de1 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2692,7 +2692,7 @@ checkRedirectedNowhere params token = case drop 1 $ getPath (parentMap params) t of T_DollarExpansion _ [_] : _ -> True T_Backticked _ [_] : _ -> True - T_Annotation _ _ u : _ -> isInExpansion u + t@T_Annotation {} : _ -> isInExpansion t _ -> False getDanglingRedirect token = case token of diff --git a/src/ShellCheck/Checker.hs b/src/ShellCheck/Checker.hs index 7ac9c91..10074e3 100644 --- a/src/ShellCheck/Checker.hs +++ b/src/ShellCheck/Checker.hs @@ -231,5 +231,7 @@ prop_filewideAnnotation8 = null $ prop_sourcePartOfOriginalScript = -- #1181: -x disabled posix warning for 'source' 2039 `elem` checkWithIncludes [("./saywhat.sh", "echo foo")] "#!/bin/sh\nsource ./saywhat.sh" +prop_spinBug1413 = null $ check "fun() {\n# shellcheck disable=SC2188\n> /dev/null\n}\n" + return [] runTests = $quickCheckAll From 3d61b73e919acda00ae2a7fd1e052b3a9280d6e1 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 16 Dec 2018 15:14:29 -0800 Subject: [PATCH 026/763] Be more specific about why you should read the wiki page --- .github/ISSUE_TEMPLATE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 4fc3f9f..b7f88e8 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -1,7 +1,7 @@ #### For bugs - Rule Id (if any, e.g. SC1000): - My shellcheck version (`shellcheck --version` or "online"): -- [ ] I read the issue's wiki page, e.g. https://github.com/koalaman/shellcheck/wiki/SC2086 +- [ ] The rule's wiki page does not already cover this (e.g. https://shellcheck.net/wiki/SC2086) - [ ] I tried on shellcheck.net and verified that this is still a problem on the latest commit #### For new checks and feature suggestions From 88aef838f127f8a6ddc767f0d6b6b2fd22db5a24 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 16 Dec 2018 15:45:52 -0800 Subject: [PATCH 027/763] SC1068 (var = x) now alternatively suggests quoting (fixes #1412) --- src/ShellCheck/Parser.hs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 3330979..172ef54 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -2715,9 +2715,10 @@ readAssignmentWordExt lenient = try $ do when (hasLeftSpace || hasRightSpace) $ parseNoteAt pos ErrorC 1068 $ "Don't put spaces around the " - ++ if op == Append - then "+= when appending." - else "= in assignments." + ++ (if op == Append + then "+= when appending" + else "= in assignments") + ++ " (or quote to make it literal)." value <- readArray <|> readNormalWord spacing return $ T_Assignment id op variable indices value From d5ba41035bf4ea42b7d7a05e928cf1a9fc0c98c8 Mon Sep 17 00:00:00 2001 From: Ng Zhi An Date: Sun, 28 Oct 2018 11:03:58 -0700 Subject: [PATCH 028/763] Add method to apply a multi-line replacement --- src/ShellCheck/Formatter/TTY.hs | 37 +++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/src/ShellCheck/Formatter/TTY.hs b/src/ShellCheck/Formatter/TTY.hs index 2d6c010..bbf01d2 100644 --- a/src/ShellCheck/Formatter/TTY.hs +++ b/src/ShellCheck/Formatter/TTY.hs @@ -183,6 +183,43 @@ doReplace start end o r = in x ++ r ++ z +-- A replacement that spans multiple line is applied by: +-- 1. merging the affected lines into a single string using `unlines` +-- 2. apply the replacement as if it only spanned a single line +-- The tricky part is adjusting the end column of the replacement +-- (the end line doesn't matter because there is only one line) +-- +-- aaS <--- start of replacement (row 1 column 3) +-- bbbb +-- cEc +-- \------- end of replacement (row 3 column 2) +-- +-- a flattened string will look like: +-- +-- "aaS\nbbbb\ncEc\n" +-- +-- The column of E has to be adjusted by: +-- 1. lengths of lines to be replaced, except the end row itself +-- 2. end column of the replacement +-- 3. number of '\n' by `unlines` +-- Returns the original lines from the file with the replacement applied. +-- Multiline replacements completely overwrite new lines in the original string. +-- e.g. if the replacement spans 2 lines, but the replacement string does not +-- have a '\n', then the number of replaced lines will be 1 shorter. +replaceMultiLines fileLines rep = + let startRow = fromIntegral $ (posLine . repStartPos) rep + endRow = fromIntegral $ (posLine . repEndPos) rep + (ys, zs) = splitAt endRow fileLines + (xs, toReplaceLines) = splitAt (startRow-1) ys + lengths = fromIntegral $ sum (map length (init toReplaceLines)) + newlines = fromIntegral $ (length toReplaceLines - 1) -- for the '\n' from unlines + original = unlines toReplaceLines + startCol = ((posColumn . repStartPos) rep) + endCol = ((posColumn . repEndPos) rep + newlines + lengths) + replacedLines = (lines $ doReplace startCol endCol original (repString rep)) + in + xs ++ replacedLines ++ zs + cuteIndent :: PositionedComment -> String cuteIndent comment = replicate (fromIntegral $ colNo comment - 1) ' ' ++ From 3471ad45b18c2a70132b5ddd28652a66f4d3f398 Mon Sep 17 00:00:00 2001 From: Ng Zhi An Date: Fri, 2 Nov 2018 22:13:49 -0700 Subject: [PATCH 029/763] Smarter sorting and application of fix to handle multiple replacements --- src/ShellCheck/Analytics.hs | 6 +-- src/ShellCheck/Formatter/TTY.hs | 85 ++++++++++++++++++++------------- src/ShellCheck/Interface.hs | 5 +- 3 files changed, 57 insertions(+), 39 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index bbd7453..e86341d 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -255,15 +255,13 @@ replaceStart id params n r = repString = r } replaceEnd id params n r = - -- because of the way we count columns 1-based - -- we have to offset end columns by 1 let tp = tokenPositions params (_, end) = tp Map.! id new_start = end { - posColumn = posColumn end - n + 1 + posColumn = posColumn end - n } new_end = end { - posColumn = posColumn end + 1 + posColumn = posColumn end } in newReplacement { diff --git a/src/ShellCheck/Formatter/TTY.hs b/src/ShellCheck/Formatter/TTY.hs index bbf01d2..1afeb1a 100644 --- a/src/ShellCheck/Formatter/TTY.hs +++ b/src/ShellCheck/Formatter/TTY.hs @@ -130,7 +130,7 @@ outputForFile color sys comments = do putStrLn (color "source" line) mapM_ (\c -> putStrLn (color (severityText c) $ cuteIndent c)) commentsForLine putStrLn "" - showFixedString color comments lineNum line + showFixedString color comments lineNum fileLines ) groups hasApplicableFix lineNum comment = fromMaybe False $ do @@ -141,47 +141,43 @@ hasApplicableFix lineNum comment = fromMaybe False $ do onSameLine pos = posLine pos == lineNum -- FIXME: Work correctly with multiple replacements -showFixedString color comments lineNum line = +showFixedString color comments lineNum fileLines = + let line = fileLines !! fromIntegral (lineNum - 1) in + -- need to check overlaps case filter (hasApplicableFix lineNum) comments of (first:_) -> do -- in the spirit of error prone putStrLn $ color "message" "Did you mean: " - putStrLn $ fixedString first line + putStrLn $ unlines $ fixedString first fileLines putStrLn "" _ -> return () --- need to do something smart about sorting by end index -fixedString :: PositionedComment -> String -> String -fixedString comment line = +fixedString :: PositionedComment -> [String] -> [String] +fixedString comment fileLines = + let lineNum = lineNo comment + line = fileLines !! fromIntegral (lineNum - 1) in case (pcFix comment) of - Nothing -> "" + Nothing -> [""] Just rs -> - applyReplacement (fixReplacements rs) line 0 + -- apply replacements in sorted order by end position + -- assert no overlaps, or maybe remove overlaps? + let sorted = (reverse . sort) (fixReplacements rs) + (start, end) = calculateOverlap sorted 1 1 + in + -- applyReplacement returns the full update file, we really only care about the changed lines + -- so we calculate overlapping lines using replacements + -- TODO separate this logic of printing affected lines out + -- since for some output we might want to have the full file output + drop (fromIntegral start) $ take (fromIntegral end) $ applyReplacement sorted fileLines where - applyReplacement [] s _ = s - applyReplacement (rep:xs) s offset = - let replacementString = repString rep - start = (posColumn . repStartPos) rep - end = (posColumn . repEndPos) rep - z = doReplace start end s replacementString - len_r = (fromIntegral . length) replacementString in - applyReplacement xs z (offset + (end - start) + len_r) - --- FIXME: Work correctly with tabs --- start and end comes from pos, which is 1 based --- doReplace 0 0 "1234" "A" -> "A1234" -- technically not valid --- doReplace 1 1 "1234" "A" -> "A1234" --- doReplace 1 2 "1234" "A" -> "A234" --- doReplace 3 3 "1234" "A" -> "12A34" --- doReplace 4 4 "1234" "A" -> "123A4" --- doReplace 5 5 "1234" "A" -> "1234A" -doReplace start end o r = - let si = fromIntegral (start-1) - ei = fromIntegral (end-1) - (x, xs) = splitAt si o - (y, z) = splitAt (ei - si) xs - in - x ++ r ++ z + applyReplacement [] s = s + applyReplacement (rep:xs) s = + let result = replaceMultiLines rep s + in + applyReplacement xs result + calculateOverlap [] s e = (s, e) + calculateOverlap (rep:xs) s e = + calculateOverlap xs (min s (posLine (repStartPos rep))) (max e (posLine (repEndPos rep))) -- A replacement that spans multiple line is applied by: -- 1. merging the affected lines into a single string using `unlines` @@ -206,9 +202,9 @@ doReplace start end o r = -- Multiline replacements completely overwrite new lines in the original string. -- e.g. if the replacement spans 2 lines, but the replacement string does not -- have a '\n', then the number of replaced lines will be 1 shorter. -replaceMultiLines fileLines rep = +replaceMultiLines rep fileLines = -- this can replace doReplace let startRow = fromIntegral $ (posLine . repStartPos) rep - endRow = fromIntegral $ (posLine . repEndPos) rep + endRow = fromIntegral $ (posLine . repEndPos) rep (ys, zs) = splitAt endRow fileLines (xs, toReplaceLines) = splitAt (startRow-1) ys lengths = fromIntegral $ sum (map length (init toReplaceLines)) @@ -220,6 +216,27 @@ replaceMultiLines fileLines rep = in xs ++ replacedLines ++ zs +-- FIXME: Work correctly with tabs +-- start and end comes from pos, which is 1 based +-- doReplace 0 0 "1234" "A" -> "A1234" -- technically not valid +-- doReplace 1 1 "1234" "A" -> "A1234" +-- doReplace 1 2 "1234" "A" -> "A234" +-- doReplace 3 3 "1234" "A" -> "12A34" +-- doReplace 4 4 "1234" "A" -> "123A4" +-- doReplace 5 5 "1234" "A" -> "1234A" +doReplace start end o r = + let si = fromIntegral (start-1) + ei = fromIntegral (end-1) + (x, xs) = splitAt si o + (y, z) = splitAt (ei - si) xs + in + x ++ r ++ z + +start = newPosition { posLine = 2, posColumn = 3 } +end = newPosition { posLine = 2, posColumn = 4 } +r = newReplacement { repStartPos = start, repEndPos = end, repString = "hello" } +filelines = ["first", "second", "third", "fourth"] + cuteIndent :: PositionedComment -> String cuteIndent comment = replicate (fromIntegral $ colNo comment - 1) ' ' ++ diff --git a/src/ShellCheck/Interface.hs b/src/ShellCheck/Interface.hs index 092b9e8..b473be2 100644 --- a/src/ShellCheck/Interface.hs +++ b/src/ShellCheck/Interface.hs @@ -180,7 +180,7 @@ data Position = Position { posFile :: String, -- Filename posLine :: Integer, -- 1 based source line posColumn :: Integer -- 1 based source column, where tabs are 8 -} deriving (Show, Eq, Generic, NFData) +} deriving (Show, Eq, Generic, NFData, Ord) newPosition :: Position newPosition = Position { @@ -209,6 +209,9 @@ data Replacement = Replacement { repString :: String } deriving (Show, Eq, Generic, NFData) +instance Ord Replacement where + compare r1 r2 = (repStartPos r1) `compare` (repStartPos r2) + newReplacement = Replacement { repStartPos = newPosition, repEndPos = newPosition, From bc111141f82cfb8a9648d4fa98a539854e76754f Mon Sep 17 00:00:00 2001 From: Ng Zhi An Date: Fri, 2 Nov 2018 22:25:51 -0700 Subject: [PATCH 030/763] Move fix application logic to separate module --- src/ShellCheck/Fixer.hs | 67 ++++++++++++++++++++++++++ src/ShellCheck/Formatter/TTY.hs | 85 ++++----------------------------- 2 files changed, 75 insertions(+), 77 deletions(-) create mode 100644 src/ShellCheck/Fixer.hs diff --git a/src/ShellCheck/Fixer.hs b/src/ShellCheck/Fixer.hs new file mode 100644 index 0000000..c81d072 --- /dev/null +++ b/src/ShellCheck/Fixer.hs @@ -0,0 +1,67 @@ +module ShellCheck.Fixer (applyFix , replaceMultiLines) where + +import ShellCheck.Interface + +import Data.List + +applyFix fix fileLines = + -- apply replacements in sorted order by end position + -- assert no overlaps, or maybe remove overlaps? + let sorted = (reverse . sort) (fixReplacements fix) in + applyReplacement sorted fileLines + where + applyReplacement [] s = s + applyReplacement (rep:xs) s = applyReplacement xs $ replaceMultiLines rep s + +-- A replacement that spans multiple line is applied by: +-- 1. merging the affected lines into a single string using `unlines` +-- 2. apply the replacement as if it only spanned a single line +-- The tricky part is adjusting the end column of the replacement +-- (the end line doesn't matter because there is only one line) +-- +-- aaS <--- start of replacement (row 1 column 3) +-- bbbb +-- cEc +-- \------- end of replacement (row 3 column 2) +-- +-- a flattened string will look like: +-- +-- "aaS\nbbbb\ncEc\n" +-- +-- The column of E has to be adjusted by: +-- 1. lengths of lines to be replaced, except the end row itself +-- 2. end column of the replacement +-- 3. number of '\n' by `unlines` +-- Returns the original lines from the file with the replacement applied. +-- Multiline replacements completely overwrite new lines in the original string. +-- e.g. if the replacement spans 2 lines, but the replacement string does not +-- have a '\n', then the number of replaced lines will be 1 shorter. +replaceMultiLines rep fileLines = -- this can replace doReplace + let startRow = fromIntegral $ (posLine . repStartPos) rep + endRow = fromIntegral $ (posLine . repEndPos) rep + (ys, zs) = splitAt endRow fileLines + (xs, toReplaceLines) = splitAt (startRow-1) ys + lengths = fromIntegral $ sum (map length (init toReplaceLines)) + newlines = fromIntegral $ (length toReplaceLines - 1) -- for the '\n' from unlines + original = unlines toReplaceLines + startCol = ((posColumn . repStartPos) rep) + endCol = ((posColumn . repEndPos) rep + newlines + lengths) + replacedLines = (lines $ doReplace startCol endCol original (repString rep)) + in + xs ++ replacedLines ++ zs + +-- FIXME: Work correctly with tabs +-- start and end comes from pos, which is 1 based +-- doReplace 0 0 "1234" "A" -> "A1234" -- technically not valid +-- doReplace 1 1 "1234" "A" -> "A1234" +-- doReplace 1 2 "1234" "A" -> "A234" +-- doReplace 3 3 "1234" "A" -> "12A34" +-- doReplace 4 4 "1234" "A" -> "123A4" +-- doReplace 5 5 "1234" "A" -> "1234A" +doReplace start end o r = + let si = fromIntegral (start-1) + ei = fromIntegral (end-1) + (x, xs) = splitAt si o + (y, z) = splitAt (ei - si) xs + in + x ++ r ++ z diff --git a/src/ShellCheck/Formatter/TTY.hs b/src/ShellCheck/Formatter/TTY.hs index 1afeb1a..acc1ff4 100644 --- a/src/ShellCheck/Formatter/TTY.hs +++ b/src/ShellCheck/Formatter/TTY.hs @@ -19,6 +19,7 @@ -} module ShellCheck.Formatter.TTY (format) where +import ShellCheck.Fixer import ShellCheck.Interface import ShellCheck.Formatter.Format @@ -154,88 +155,18 @@ showFixedString color comments lineNum fileLines = fixedString :: PositionedComment -> [String] -> [String] fixedString comment fileLines = - let lineNum = lineNo comment - line = fileLines !! fromIntegral (lineNum - 1) in case (pcFix comment) of Nothing -> [""] - Just rs -> - -- apply replacements in sorted order by end position - -- assert no overlaps, or maybe remove overlaps? - let sorted = (reverse . sort) (fixReplacements rs) - (start, end) = calculateOverlap sorted 1 1 - in + Just fix -> + let (start, end) = affectedRange (fixReplacements fix) in -- applyReplacement returns the full update file, we really only care about the changed lines -- so we calculate overlapping lines using replacements - -- TODO separate this logic of printing affected lines out - -- since for some output we might want to have the full file output - drop (fromIntegral start) $ take (fromIntegral end) $ applyReplacement sorted fileLines + drop start $ take end $ applyFix fix fileLines where - applyReplacement [] s = s - applyReplacement (rep:xs) s = - let result = replaceMultiLines rep s - in - applyReplacement xs result - calculateOverlap [] s e = (s, e) - calculateOverlap (rep:xs) s e = - calculateOverlap xs (min s (posLine (repStartPos rep))) (max e (posLine (repEndPos rep))) - --- A replacement that spans multiple line is applied by: --- 1. merging the affected lines into a single string using `unlines` --- 2. apply the replacement as if it only spanned a single line --- The tricky part is adjusting the end column of the replacement --- (the end line doesn't matter because there is only one line) --- --- aaS <--- start of replacement (row 1 column 3) --- bbbb --- cEc --- \------- end of replacement (row 3 column 2) --- --- a flattened string will look like: --- --- "aaS\nbbbb\ncEc\n" --- --- The column of E has to be adjusted by: --- 1. lengths of lines to be replaced, except the end row itself --- 2. end column of the replacement --- 3. number of '\n' by `unlines` --- Returns the original lines from the file with the replacement applied. --- Multiline replacements completely overwrite new lines in the original string. --- e.g. if the replacement spans 2 lines, but the replacement string does not --- have a '\n', then the number of replaced lines will be 1 shorter. -replaceMultiLines rep fileLines = -- this can replace doReplace - let startRow = fromIntegral $ (posLine . repStartPos) rep - endRow = fromIntegral $ (posLine . repEndPos) rep - (ys, zs) = splitAt endRow fileLines - (xs, toReplaceLines) = splitAt (startRow-1) ys - lengths = fromIntegral $ sum (map length (init toReplaceLines)) - newlines = fromIntegral $ (length toReplaceLines - 1) -- for the '\n' from unlines - original = unlines toReplaceLines - startCol = ((posColumn . repStartPos) rep) - endCol = ((posColumn . repEndPos) rep + newlines + lengths) - replacedLines = (lines $ doReplace startCol endCol original (repString rep)) - in - xs ++ replacedLines ++ zs - --- FIXME: Work correctly with tabs --- start and end comes from pos, which is 1 based --- doReplace 0 0 "1234" "A" -> "A1234" -- technically not valid --- doReplace 1 1 "1234" "A" -> "A1234" --- doReplace 1 2 "1234" "A" -> "A234" --- doReplace 3 3 "1234" "A" -> "12A34" --- doReplace 4 4 "1234" "A" -> "123A4" --- doReplace 5 5 "1234" "A" -> "1234A" -doReplace start end o r = - let si = fromIntegral (start-1) - ei = fromIntegral (end-1) - (x, xs) = splitAt si o - (y, z) = splitAt (ei - si) xs - in - x ++ r ++ z - -start = newPosition { posLine = 2, posColumn = 3 } -end = newPosition { posLine = 2, posColumn = 4 } -r = newReplacement { repStartPos = start, repEndPos = end, repString = "hello" } -filelines = ["first", "second", "third", "fourth"] + affectedRange rs = _affectedRange rs 1 1 + _affectedRange [] s e = (fromIntegral s, fromIntegral e) + _affectedRange (rep:xs) s e = + _affectedRange xs (min s (posLine (repStartPos rep))) (max e (posLine (repEndPos rep))) cuteIndent :: PositionedComment -> String cuteIndent comment = From 408a3b99d885387219953ae27d0309703738906f Mon Sep 17 00:00:00 2001 From: Ng Zhi An Date: Wed, 7 Nov 2018 22:06:43 -0800 Subject: [PATCH 031/763] Remove overlaps before applying replacements --- src/ShellCheck/Fixer.hs | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Fixer.hs b/src/ShellCheck/Fixer.hs index c81d072..1cabb1e 100644 --- a/src/ShellCheck/Fixer.hs +++ b/src/ShellCheck/Fixer.hs @@ -6,12 +6,26 @@ import Data.List applyFix fix fileLines = -- apply replacements in sorted order by end position - -- assert no overlaps, or maybe remove overlaps? - let sorted = (reverse . sort) (fixReplacements fix) in + let sorted = (removeOverlap . reverse . sort) (fixReplacements fix) in applyReplacement sorted fileLines where applyReplacement [] s = s applyReplacement (rep:xs) s = applyReplacement xs $ replaceMultiLines rep s + -- prereq: list is already sorted by start position + removeOverlap [] = [] + removeOverlap (x:xs) = checkoverlap x xs + checkoverlap :: Replacement -> [Replacement] -> [Replacement] + checkoverlap x [] = x:[] + checkoverlap x (y:ys) = + if overlap x y then x:(removeOverlap ys) else x:y:(removeOverlap ys) + -- two position overlaps when + overlap x y = + (yStart >= xStart && yStart <= xEnd) || (yStart < xStart && yStart > xStart) + where + yStart = repStartPos y + xStart = repStartPos x + xEnd = repEndPos x + -- A replacement that spans multiple line is applied by: -- 1. merging the affected lines into a single string using `unlines` From 3403f8d75bd4de6199b51c4c8637b9ea57d435aa Mon Sep 17 00:00:00 2001 From: Ng Zhi An Date: Sun, 16 Dec 2018 11:30:50 -0800 Subject: [PATCH 032/763] Fix bug in overlap check --- src/ShellCheck/Fixer.hs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Fixer.hs b/src/ShellCheck/Fixer.hs index 1cabb1e..52a991c 100644 --- a/src/ShellCheck/Fixer.hs +++ b/src/ShellCheck/Fixer.hs @@ -20,9 +20,10 @@ applyFix fix fileLines = if overlap x y then x:(removeOverlap ys) else x:y:(removeOverlap ys) -- two position overlaps when overlap x y = - (yStart >= xStart && yStart <= xEnd) || (yStart < xStart && yStart > xStart) + (yStart >= xStart && yStart < xEnd) || (yStart < xStart && yEnd > xStart) where yStart = repStartPos y + yEnd = repEndPos y xStart = repStartPos x xEnd = repEndPos x From 7d2c519d64c233962fc449ab80a287f7c2206121 Mon Sep 17 00:00:00 2001 From: Ng Zhi An Date: Sun, 16 Dec 2018 22:17:44 -0800 Subject: [PATCH 033/763] Remove spurious new line in fix message --- src/ShellCheck/Formatter/TTY.hs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/ShellCheck/Formatter/TTY.hs b/src/ShellCheck/Formatter/TTY.hs index acc1ff4..3e0a2a8 100644 --- a/src/ShellCheck/Formatter/TTY.hs +++ b/src/ShellCheck/Formatter/TTY.hs @@ -150,7 +150,6 @@ showFixedString color comments lineNum fileLines = -- in the spirit of error prone putStrLn $ color "message" "Did you mean: " putStrLn $ unlines $ fixedString first fileLines - putStrLn "" _ -> return () fixedString :: PositionedComment -> [String] -> [String] From a8d88dfe980a597add7ca07787350775131cd76c Mon Sep 17 00:00:00 2001 From: Ng Zhi An Date: Mon, 17 Dec 2018 00:19:49 -0800 Subject: [PATCH 034/763] Fix calculation of changed lines --- src/ShellCheck/Formatter/TTY.hs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/ShellCheck/Formatter/TTY.hs b/src/ShellCheck/Formatter/TTY.hs index 3e0a2a8..dfe5421 100644 --- a/src/ShellCheck/Formatter/TTY.hs +++ b/src/ShellCheck/Formatter/TTY.hs @@ -24,6 +24,7 @@ import ShellCheck.Interface import ShellCheck.Formatter.Format import Control.Monad +import Data.Ord import Data.IORef import Data.List import Data.Maybe @@ -156,16 +157,15 @@ fixedString :: PositionedComment -> [String] -> [String] fixedString comment fileLines = case (pcFix comment) of Nothing -> [""] - Just fix -> - let (start, end) = affectedRange (fixReplacements fix) in - -- applyReplacement returns the full update file, we really only care about the changed lines - -- so we calculate overlapping lines using replacements - drop start $ take end $ applyFix fix fileLines - where - affectedRange rs = _affectedRange rs 1 1 - _affectedRange [] s e = (fromIntegral s, fromIntegral e) - _affectedRange (rep:xs) s e = - _affectedRange xs (min s (posLine (repStartPos rep))) (max e (posLine (repEndPos rep))) + Just fix -> case (fixReplacements fix) of + [] -> [] + reps -> + -- applyReplacement returns the full update file, we really only care about the changed lines + -- so we calculate overlapping lines using replacements + drop start $ take end $ applyFix fix fileLines + where + start = (fromIntegral $ minimum $ map (posLine . repStartPos) reps) - 1 + end = fromIntegral $ maximum $ map (posLine . repEndPos) reps cuteIndent :: PositionedComment -> String cuteIndent comment = From eb3e6fe8e1e0ef824a01daa1b8405845af127158 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 17 Dec 2018 20:14:49 -0800 Subject: [PATCH 035/763] Add ShellCheck.Fixer to the cabal file --- ShellCheck.cabal | 1 + 1 file changed, 1 insertion(+) diff --git a/ShellCheck.cabal b/ShellCheck.cabal index 721da3f..adf5c7d 100644 --- a/ShellCheck.cabal +++ b/ShellCheck.cabal @@ -73,6 +73,7 @@ library ShellCheck.Checks.Commands ShellCheck.Checks.ShellSupport ShellCheck.Data + ShellCheck.Fixer ShellCheck.Formatter.Format ShellCheck.Formatter.CheckStyle ShellCheck.Formatter.GCC From 08ca1ee6e915487a304a8c45c279d180240cef16 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 17 Dec 2018 20:15:39 -0800 Subject: [PATCH 036/763] Remove unnecessary Regex constraint --- src/ShellCheck/Regex.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Regex.hs b/src/ShellCheck/Regex.hs index f975775..f1262b4 100644 --- a/src/ShellCheck/Regex.hs +++ b/src/ShellCheck/Regex.hs @@ -30,7 +30,7 @@ import Text.Regex.TDFA -- Precompile the regex mkRegex :: String -> Regex mkRegex str = - let make :: RegexMaker Regex CompOption ExecOption String => String -> Regex + let make :: String -> Regex make = makeRegex in make str From 0636e7023c4257f9f2ef5892db27a81f1422db43 Mon Sep 17 00:00:00 2001 From: Ng Zhi An Date: Fri, 21 Dec 2018 14:34:03 +0800 Subject: [PATCH 037/763] Fix applying multiple fixes per line Fixes #1421 --- src/ShellCheck/Fixer.hs | 9 -------- src/ShellCheck/Formatter/TTY.hs | 19 ++++++++++------- src/ShellCheck/Interface.hs | 37 +++++++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 17 deletions(-) diff --git a/src/ShellCheck/Fixer.hs b/src/ShellCheck/Fixer.hs index 52a991c..a393cec 100644 --- a/src/ShellCheck/Fixer.hs +++ b/src/ShellCheck/Fixer.hs @@ -14,18 +14,9 @@ applyFix fix fileLines = -- prereq: list is already sorted by start position removeOverlap [] = [] removeOverlap (x:xs) = checkoverlap x xs - checkoverlap :: Replacement -> [Replacement] -> [Replacement] checkoverlap x [] = x:[] checkoverlap x (y:ys) = if overlap x y then x:(removeOverlap ys) else x:y:(removeOverlap ys) - -- two position overlaps when - overlap x y = - (yStart >= xStart && yStart < xEnd) || (yStart < xStart && yEnd > xStart) - where - yStart = repStartPos y - yEnd = repEndPos y - xStart = repStartPos x - xEnd = repEndPos x -- A replacement that spans multiple line is applied by: diff --git a/src/ShellCheck/Formatter/TTY.hs b/src/ShellCheck/Formatter/TTY.hs index dfe5421..adcc277 100644 --- a/src/ShellCheck/Formatter/TTY.hs +++ b/src/ShellCheck/Formatter/TTY.hs @@ -24,6 +24,7 @@ import ShellCheck.Interface import ShellCheck.Formatter.Format import Control.Monad +import Data.Foldable import Data.Ord import Data.IORef import Data.List @@ -147,17 +148,19 @@ showFixedString color comments lineNum fileLines = let line = fileLines !! fromIntegral (lineNum - 1) in -- need to check overlaps case filter (hasApplicableFix lineNum) comments of - (first:_) -> do + [] -> return () + -- all the fixes are single-line only, but there could be multiple + -- fixes for that single line. We can fold the fixes (which removes + -- overlaps), and apply it as a single fix with multiple replacements. + applicableComments -> do + let mergedFix = (fold . catMaybes . (map pcFix)) applicableComments -- in the spirit of error prone putStrLn $ color "message" "Did you mean: " - putStrLn $ unlines $ fixedString first fileLines - _ -> return () + putStrLn $ unlines $ fixedString mergedFix fileLines -fixedString :: PositionedComment -> [String] -> [String] -fixedString comment fileLines = - case (pcFix comment) of - Nothing -> [""] - Just fix -> case (fixReplacements fix) of +fixedString :: Fix -> [String] -> [String] +fixedString fix fileLines = + case (fixReplacements fix) of [] -> [] reps -> -- applyReplacement returns the full update file, we really only care about the changed lines diff --git a/src/ShellCheck/Interface.hs b/src/ShellCheck/Interface.hs index b473be2..c0bf79f 100644 --- a/src/ShellCheck/Interface.hs +++ b/src/ShellCheck/Interface.hs @@ -54,13 +54,17 @@ module ShellCheck.Interface , newFix , Replacement(repStartPos, repEndPos, repString) , newReplacement + , Ranged(overlap) ) where import ShellCheck.AST import Control.DeepSeq import Control.Monad.Identity +import Data.List import Data.Monoid +import Data.Ord +import Data.Semigroup import GHC.Generics (Generic) import qualified Data.Map as Map @@ -270,3 +274,36 @@ mockedSystemInterface files = SystemInterface { [] -> return $ Left "File not included in mock." [(_, contents)] -> return $ Right contents +-- The Ranged class is used for types that has a start and end position. +class Ranged a where + start :: a -> Position + end :: a -> Position + overlap :: a -> a -> Bool + overlap x y = + (yStart >= xStart && yStart < xEnd) || (yStart < xStart && yEnd > xStart) + where + yStart = start y + yEnd = end y + xStart = start x + xEnd = end x + +instance Ranged Replacement where + start = repStartPos + end = repEndPos + +instance Ranged a => Ranged [a] where + start [] = newPosition + start xs = (minimum . map start) xs + end [] = newPosition + end xs = (maximum . map end) xs + +instance Ranged Fix where + start = start . fixReplacements + end = end . fixReplacements + +-- The Monoid instance for Fix merges replacements that do not overlap. +instance Monoid Fix where + mempty = newFix + f1 `mappend` f2 = if overlap f1 f2 then f1 else newFix { + fixReplacements = fixReplacements f1 ++ fixReplacements f2 + } From 897f019353f29c4751359c76675db071bf6008fd Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 22 Dec 2018 10:04:00 -0800 Subject: [PATCH 038/763] Move Ranged definition to Fixer to avoid overpromising --- src/ShellCheck/Fixer.hs | 39 +++++++++++++++++++++++++++++++++++-- src/ShellCheck/Interface.hs | 34 -------------------------------- 2 files changed, 37 insertions(+), 36 deletions(-) diff --git a/src/ShellCheck/Fixer.hs b/src/ShellCheck/Fixer.hs index a393cec..75b6fdd 100644 --- a/src/ShellCheck/Fixer.hs +++ b/src/ShellCheck/Fixer.hs @@ -1,9 +1,44 @@ -module ShellCheck.Fixer (applyFix , replaceMultiLines) where +module ShellCheck.Fixer (applyFix , replaceMultiLines, Ranged(..)) where import ShellCheck.Interface - import Data.List +-- The Ranged class is used for types that has a start and end position. +class Ranged a where + start :: a -> Position + end :: a -> Position + overlap :: a -> a -> Bool + overlap x y = + (yStart >= xStart && yStart < xEnd) || (yStart < xStart && yEnd > xStart) + where + yStart = start y + yEnd = end y + xStart = start x + xEnd = end x + +instance Ranged Replacement where + start = repStartPos + end = repEndPos + +instance Ranged a => Ranged [a] where + start [] = newPosition + start xs = (minimum . map start) xs + end [] = newPosition + end xs = (maximum . map end) xs + +instance Ranged Fix where + start = start . fixReplacements + end = end . fixReplacements + +-- The Monoid instance for Fix merges replacements that do not overlap. +instance Monoid Fix where + mempty = newFix + +instance Semigroup Fix where + f1 <> f2 = if overlap f1 f2 then f1 else newFix { + fixReplacements = fixReplacements f1 ++ fixReplacements f2 + } + applyFix fix fileLines = -- apply replacements in sorted order by end position let sorted = (removeOverlap . reverse . sort) (fixReplacements fix) in diff --git a/src/ShellCheck/Interface.hs b/src/ShellCheck/Interface.hs index c0bf79f..ea70c15 100644 --- a/src/ShellCheck/Interface.hs +++ b/src/ShellCheck/Interface.hs @@ -54,7 +54,6 @@ module ShellCheck.Interface , newFix , Replacement(repStartPos, repEndPos, repString) , newReplacement - , Ranged(overlap) ) where import ShellCheck.AST @@ -274,36 +273,3 @@ mockedSystemInterface files = SystemInterface { [] -> return $ Left "File not included in mock." [(_, contents)] -> return $ Right contents --- The Ranged class is used for types that has a start and end position. -class Ranged a where - start :: a -> Position - end :: a -> Position - overlap :: a -> a -> Bool - overlap x y = - (yStart >= xStart && yStart < xEnd) || (yStart < xStart && yEnd > xStart) - where - yStart = start y - yEnd = end y - xStart = start x - xEnd = end x - -instance Ranged Replacement where - start = repStartPos - end = repEndPos - -instance Ranged a => Ranged [a] where - start [] = newPosition - start xs = (minimum . map start) xs - end [] = newPosition - end xs = (maximum . map end) xs - -instance Ranged Fix where - start = start . fixReplacements - end = end . fixReplacements - --- The Monoid instance for Fix merges replacements that do not overlap. -instance Monoid Fix where - mempty = newFix - f1 `mappend` f2 = if overlap f1 f2 then f1 else newFix { - fixReplacements = fixReplacements f1 ++ fixReplacements f2 - } From 9acc8fcb533c544141725c14cfcbbd235f991ec3 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 23 Dec 2018 11:08:48 -0800 Subject: [PATCH 039/763] Fix semigroup incompatibility --- src/ShellCheck/Fixer.hs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ShellCheck/Fixer.hs b/src/ShellCheck/Fixer.hs index 75b6fdd..78abfeb 100644 --- a/src/ShellCheck/Fixer.hs +++ b/src/ShellCheck/Fixer.hs @@ -2,6 +2,7 @@ module ShellCheck.Fixer (applyFix , replaceMultiLines, Ranged(..)) where import ShellCheck.Interface import Data.List +import Data.Semigroup -- The Ranged class is used for types that has a start and end position. class Ranged a where @@ -33,6 +34,7 @@ instance Ranged Fix where -- The Monoid instance for Fix merges replacements that do not overlap. instance Monoid Fix where mempty = newFix + mappend = (<>) instance Semigroup Fix where f1 <> f2 = if overlap f1 f2 then f1 else newFix { From bd04af0769c3ca643bff0920c27028a6cc84fe02 Mon Sep 17 00:00:00 2001 From: Ng Zhi An Date: Sat, 22 Dec 2018 22:16:24 +0800 Subject: [PATCH 040/763] Update supported ulimit flags for dash Values are retrieved from https://linux.die.net/man/1/dash, search for ulimit. Fixes #1406 --- src/ShellCheck/Checks/ShellSupport.hs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 1bc425f..02f2bfa 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -136,6 +136,8 @@ prop_checkBashisms53= verifyNot checkBashisms "#!/bin/sh\nprintf -- -f\n" prop_checkBashisms54= verify checkBashisms "#!/bin/sh\nfoo+=bar" prop_checkBashisms55= verify checkBashisms "#!/bin/sh\necho ${@%foo}" prop_checkBashisms56= verifyNot checkBashisms "#!/bin/sh\necho ${##}" +prop_checkBashisms57= verifyNot checkBashisms "#!/bin/dash\nulimit -c 0" +prop_checkBashisms58= verify checkBashisms "#!/bin/sh\nulimit -c 0" checkBashisms = ForShell [Sh, Dash] $ \t -> do params <- ask kludge params t @@ -283,7 +285,7 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do ("export", ["-p"]), ("printf", []), ("read", if isDash then ["r", "p"] else ["r"]), - ("ulimit", ["f"]) + ("ulimit", if isDash then ["H", "S", "t", "f", "d", "s", "c", "m", "l", "p", "n"] else ["f"]) ] bashism t@(T_SourceCommand id src _) = let name = fromMaybe "" $ getCommandName src From 95a8cf93c90b5b8a215cd20c4d79ea1032b86acc Mon Sep 17 00:00:00 2001 From: Ng Zhi An Date: Sat, 22 Dec 2018 21:59:38 +0800 Subject: [PATCH 041/763] Add check for ambiguous nullary test Given an input like `if [[ $(a) ]]; then ...`, this is a implicit `-n` test, so it works like `if [[ -n $(a) ]]; then ...`. Users might confuse this for a check for the exit code of the command a, which should be tested with: if a; then ... We warn the user to be more explicity and specifity the `-n`. Fixes #1416 --- src/ShellCheck/Analytics.hs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 9b407f8..5b8d1bd 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -170,6 +170,7 @@ nodeChecks = [ ,checkSubshelledTests ,checkInvertedStringTest ,checkRedirectionToCommand + ,checkNullaryExpansionTest ] @@ -3096,5 +3097,21 @@ checkRedirectionToCommand _ t = warn id 2238 "Redirecting to/from command name instead of file. Did you want pipes/xargs (or quote to ignore)?" _ -> return () +prop_checkNullaryExpansionTest1 = verify checkNullaryExpansionTest "[[ $(a) ]]" +prop_checkNullaryExpansionTest2 = verify checkNullaryExpansionTest "[[ $a ]]" +prop_checkNullaryExpansionTest3 = verifyNot checkNullaryExpansionTest "[[ $a=1 ]]" +prop_checkNullaryExpansionTest4 = verifyNot checkNullaryExpansionTest "[[ -n $(a) ]]" +checkNullaryExpansionTest _ t = + case t of + TC_Nullary _ _ (T_NormalWord id [T_DollarExpansion _ [T_Pipeline _ [] [x]]]) -> + when (isJust (getCommand x)) $ + style id 2243 ( + "To check for the exit code of a command, remove the conditional expression, e.g. if foo; ...") + TC_Nullary _ _ (T_NormalWord id [t2]) -> + when ((not . isConstant) t2) $ + style id 2244 ( + "Use -n to check for null string, e.g. [[ -n $var ]].") + _ -> return () + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) From 73a41cdd2febac98cf0dea35decf1a56b89e22de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristian=20Adri=C3=A1n=20Ontivero?= Date: Fri, 28 Dec 2018 00:23:17 -0300 Subject: [PATCH 042/763] Check jobs flags in dash/POSIX sh (fixes #1429) --- src/ShellCheck/Checks/ShellSupport.hs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 02f2bfa..204c641 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -138,6 +138,10 @@ prop_checkBashisms55= verify checkBashisms "#!/bin/sh\necho ${@%foo}" prop_checkBashisms56= verifyNot checkBashisms "#!/bin/sh\necho ${##}" prop_checkBashisms57= verifyNot checkBashisms "#!/bin/dash\nulimit -c 0" prop_checkBashisms58= verify checkBashisms "#!/bin/sh\nulimit -c 0" +prop_checkBashisms59 = verify checkBashisms "#!/bin/sh\njobs -s" +prop_checkBashisms60 = verifyNot checkBashisms "#!/bin/sh\njobs -p" +prop_checkBashisms61 = verifyNot checkBashisms "#!/bin/sh\njobs -lp" + checkBashisms = ForShell [Sh, Dash] $ \t -> do params <- ask kludge params t @@ -283,6 +287,7 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do allowedFlags = Map.fromList [ ("exec", []), ("export", ["-p"]), + ("jobs", ["l", "p"]), ("printf", []), ("read", if isDash then ["r", "p"] else ["r"]), ("ulimit", if isDash then ["H", "S", "t", "f", "d", "s", "c", "m", "l", "p", "n"] else ["f"]) From 29dedbdc9ceb47057bb275d26b5747c976522372 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristian=20Adri=C3=A1n=20Ontivero?= Date: Fri, 28 Dec 2018 20:55:52 -0300 Subject: [PATCH 043/763] Fix 'export -p' being undefined under POSIX sh Fixes #1432 --- src/ShellCheck/Checks/ShellSupport.hs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 204c641..4935c05 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -141,6 +141,8 @@ prop_checkBashisms58= verify checkBashisms "#!/bin/sh\nulimit -c 0" prop_checkBashisms59 = verify checkBashisms "#!/bin/sh\njobs -s" prop_checkBashisms60 = verifyNot checkBashisms "#!/bin/sh\njobs -p" prop_checkBashisms61 = verifyNot checkBashisms "#!/bin/sh\njobs -lp" +prop_checkBashisms62 = verify checkBashisms "#!/bin/sh\nexport -f foo" +prop_checkBashisms63 = verifyNot checkBashisms "#!/bin/sh\nexport -p" checkBashisms = ForShell [Sh, Dash] $ \t -> do params <- ask @@ -286,7 +288,7 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do ] ++ if not isDash then ["local"] else [] allowedFlags = Map.fromList [ ("exec", []), - ("export", ["-p"]), + ("export", ["p"]), ("jobs", ["l", "p"]), ("printf", []), ("read", if isDash then ["r", "p"] else ["r"]), From 73822c35887c1d54f50ebafa1a9de7425228ffab Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Fri, 28 Dec 2018 19:01:31 -0800 Subject: [PATCH 044/763] Allow SC2243 and SC2244 to trigger with quotes, add fix --- CHANGELOG.md | 1 + src/ShellCheck/ASTLib.hs | 5 ++++- src/ShellCheck/Analytics.hs | 24 +++++++++++++++--------- src/ShellCheck/AnalyzerLib.hs | 13 ++++++++----- 4 files changed, 28 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c239e7..11e1994 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## Since previous release ### Added - Preliminary support for fix suggestions +- SC2243/SC2244: Suggest using explicit -n for `[ $foo ]` ## v0.6.0 - 2018-12-02 ### Added diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index ef72b09..5f3b68c 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -485,8 +485,11 @@ wordsCanBeEqual x y = fromMaybe True $ -- Is this an expansion that can be quoted, -- e.g. $(foo) `foo` $foo (but not {foo,})? isQuoteableExpansion t = case t of + T_DollarBraced {} -> True + _ -> isCommandSubstitution t + +isCommandSubstitution t = case t of T_DollarExpansion {} -> True T_DollarBraceCommandExpansion {} -> True T_Backticked {} -> True - T_DollarBraced {} -> True _ -> False diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 5b8d1bd..21f429f 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -3101,16 +3101,22 @@ prop_checkNullaryExpansionTest1 = verify checkNullaryExpansionTest "[[ $(a) ]]" prop_checkNullaryExpansionTest2 = verify checkNullaryExpansionTest "[[ $a ]]" prop_checkNullaryExpansionTest3 = verifyNot checkNullaryExpansionTest "[[ $a=1 ]]" prop_checkNullaryExpansionTest4 = verifyNot checkNullaryExpansionTest "[[ -n $(a) ]]" -checkNullaryExpansionTest _ t = +prop_checkNullaryExpansionTest5 = verify checkNullaryExpansionTest "[[ \"$a$b\" ]]" +prop_checkNullaryExpansionTest6 = verify checkNullaryExpansionTest "[[ `x` ]]" +checkNullaryExpansionTest params t = case t of - TC_Nullary _ _ (T_NormalWord id [T_DollarExpansion _ [T_Pipeline _ [] [x]]]) -> - when (isJust (getCommand x)) $ - style id 2243 ( - "To check for the exit code of a command, remove the conditional expression, e.g. if foo; ...") - TC_Nullary _ _ (T_NormalWord id [t2]) -> - when ((not . isConstant) t2) $ - style id 2244 ( - "Use -n to check for null string, e.g. [[ -n $var ]].") + TC_Nullary _ _ word -> + case getWordParts word of + [t] | isCommandSubstitution t -> + styleWithFix id 2243 "Prefer explicit -n to check for output (or run command without [/[[ to check for success)." fix + + -- If they're constant, you get SC2157 &co + x | all (not . isConstant) x -> + styleWithFix id 2244 "Prefer explicit -n to check non-empty string (or use =/-ne to check boolean/integer)." fix + _ -> return () + where + id = getId word + fix = fixWith [replaceStart id params 0 "-n "] _ -> return () return [] diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 306c139..90a79e0 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -155,11 +155,14 @@ err id code str = addComment $ makeComment ErrorC id code str info id code str = addComment $ makeComment InfoC id code str style id code str = addComment $ makeComment StyleC id code str -warnWithFix id code str fix = addComment $ - let comment = makeComment WarningC id code str in - comment { - tcFix = Just fix - } +warnWithFix :: MonadWriter [TokenComment] m => Id -> Code -> String -> Fix -> m () +warnWithFix = addCommentWithFix WarningC +styleWithFix :: MonadWriter [TokenComment] m => Id -> Code -> String -> Fix -> m () +styleWithFix = addCommentWithFix StyleC + +addCommentWithFix :: MonadWriter [TokenComment] m => Severity -> Id -> Code -> String -> Fix -> m () +addCommentWithFix severity id code str fix = + addComment $ makeCommentWithFix severity id code str fix makeCommentWithFix :: Severity -> Id -> Code -> String -> Fix -> TokenComment makeCommentWithFix severity id code str fix = From 461be74976ef360c0bccff4328ae5e776a71a647 Mon Sep 17 00:00:00 2001 From: Ng Zhi An Date: Sat, 22 Dec 2018 15:42:28 +0800 Subject: [PATCH 045/763] Realign virtual tabs when applying fix Fix an off-by-one error, in the case that is commented `should never happen`. It happens when the end of a range is the at the end of a line. In that case we should update the real column count (probably just by +1) instead of returning it. I modified makeNonVirtual to use a helper, realign, that works on Ranged. That way we can share the code to realign a PositionedComment and also a Replacement. Fixes #1420 --- src/ShellCheck/Fixer.hs | 18 +++++++++++++++++ src/ShellCheck/Formatter/Format.hs | 32 ++++++++++++++++++++---------- src/ShellCheck/Formatter/TTY.hs | 5 ++++- 3 files changed, 43 insertions(+), 12 deletions(-) diff --git a/src/ShellCheck/Fixer.hs b/src/ShellCheck/Fixer.hs index 78abfeb..9c4837c 100644 --- a/src/ShellCheck/Fixer.hs +++ b/src/ShellCheck/Fixer.hs @@ -16,20 +16,38 @@ class Ranged a where yEnd = end y xStart = start x xEnd = end x + -- Set a new start and end position on a Ranged + setRange :: (Position, Position) -> a -> a + +instance Ranged PositionedComment where + start = pcStartPos + end = pcEndPos + setRange (s, e) pc = pc { + pcStartPos = s, + pcEndPos = e + } instance Ranged Replacement where start = repStartPos end = repEndPos + setRange (s, e) r = r { + repStartPos = s, + repEndPos = e + } instance Ranged a => Ranged [a] where start [] = newPosition start xs = (minimum . map start) xs end [] = newPosition end xs = (maximum . map end) xs + setRange (s, e) rs = map (setRange (s, e)) rs instance Ranged Fix where start = start . fixReplacements end = end . fixReplacements + setRange (s, e) f = f { + fixReplacements = setRange (s, e) (fixReplacements f) + } -- The Monoid instance for Fix merges replacements that do not overlap. instance Monoid Fix where diff --git a/src/ShellCheck/Formatter/Format.hs b/src/ShellCheck/Formatter/Format.hs index 5e46713..5c3c444 100644 --- a/src/ShellCheck/Formatter/Format.hs +++ b/src/ShellCheck/Formatter/Format.hs @@ -21,6 +21,7 @@ module ShellCheck.Formatter.Format where import ShellCheck.Data import ShellCheck.Interface +import ShellCheck.Fixer -- A formatter that carries along an arbitrary piece of data data Formatter = Formatter { @@ -51,20 +52,29 @@ makeNonVirtual comments contents = map fix comments where ls = lines contents - fix c = c { - pcStartPos = (pcStartPos c) { - posColumn = realignColumn lineNo colNo c - } - , pcEndPos = (pcEndPos c) { - posColumn = realignColumn endLineNo endColNo c - } - } + fix c = realign c ls + +-- Realign a Ranged from a tabstop of 8 to 1 +realign :: Ranged a => a -> [String] -> a +realign range ls = + let startColumn = realignColumn lineNo colNo range + endColumn = realignColumn endLineNo endColNo range + startPosition = (start range) { posColumn = startColumn } + endPosition = (end range) { posColumn = endColumn } in + setRange (startPosition, endPosition) range + where realignColumn lineNo colNo c = if lineNo c > 0 && lineNo c <= fromIntegral (length ls) then real (ls !! fromIntegral (lineNo c - 1)) 0 0 (colNo c) else colNo c real _ r v target | target <= v = r - real [] r v _ = r -- should never happen - real ('\t':rest) r v target = - real rest (r+1) (v + 8 - (v `mod` 8)) target + -- hit this case at the end of line, and if we don't hit the target + -- return real + (target - v) + real [] r v target = r + (target - v) + real ('\t':rest) r v target = real rest (r+1) (v + 8 - (v `mod` 8)) target real (_:rest) r v target = real rest (r+1) (v+1) target + lineNo = posLine . start + endLineNo = posLine . end + colNo = posColumn . start + endColNo = posColumn . end + diff --git a/src/ShellCheck/Formatter/TTY.hs b/src/ShellCheck/Formatter/TTY.hs index adcc277..ffc5e93 100644 --- a/src/ShellCheck/Formatter/TTY.hs +++ b/src/ShellCheck/Formatter/TTY.hs @@ -153,10 +153,13 @@ showFixedString color comments lineNum fileLines = -- fixes for that single line. We can fold the fixes (which removes -- overlaps), and apply it as a single fix with multiple replacements. applicableComments -> do - let mergedFix = (fold . catMaybes . (map pcFix)) applicableComments + let mergedFix = (realignFix . fold . catMaybes . (map pcFix)) applicableComments -- in the spirit of error prone putStrLn $ color "message" "Did you mean: " putStrLn $ unlines $ fixedString mergedFix fileLines + where + realignFix f = f { fixReplacements = map fix (fixReplacements f) } + fix r = realign r fileLines fixedString :: Fix -> [String] -> [String] fixedString fix fileLines = From 9425654a422baeaa9670917fcaf4a59ef7fdd5c8 Mon Sep 17 00:00:00 2001 From: Gandalf- Date: Mon, 31 Dec 2018 15:33:37 -0800 Subject: [PATCH 046/763] Expand echo + sed style warning to herestrings https://github.com/koalaman/shellcheck/issues/130 --- src/ShellCheck/Checks/ShellSupport.hs | 44 ++++++++++++++++++--------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 204c641..65bffc2 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -325,31 +325,45 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do _ -> False prop_checkEchoSed1 = verify checkEchoSed "FOO=$(echo \"$cow\" | sed 's/foo/bar/g')" +prop_checkEchoSed1b= verify checkEchoSed "FOO=$(sed 's/foo/bar/g' <<< \"$cow\")" prop_checkEchoSed2 = verify checkEchoSed "rm $(echo $cow | sed -e 's,foo,bar,')" +prop_checkEchoSed2b= verify checkEchoSed "rm $(sed -e 's,foo,bar,' <<< $cow)" checkEchoSed = ForShell [Bash, Ksh] f where + f (T_Redirecting id lefts r) = + when (any redirectHereString lefts) $ + checkSed id rcmd + where + redirectHereString :: Token -> Bool + redirectHereString t = case t of + (T_FdRedirect _ _ T_HereString{}) -> True + _ -> False + rcmd = oversimplify r + f (T_Pipeline id _ [a, b]) = when (acmd == ["echo", "${VAR}"]) $ - case bcmd of - ["sed", v] -> checkIn v - ["sed", "-e", v] -> checkIn v - _ -> return () + checkSed id bcmd where - -- This should have used backreferences, but TDFA doesn't support them - sedRe = mkRegex "^s(.)([^\n]*)g?$" - isSimpleSed s = fromMaybe False $ do - [first,rest] <- matchRegex sedRe s - let delimiters = filter (== head first) rest - guard $ length delimiters == 2 - return True - acmd = oversimplify a bcmd = oversimplify b - checkIn s = - when (isSimpleSed s) $ - style id 2001 "See if you can use ${variable//search/replace} instead." + f _ = return () + checkSed id ["sed", v] = checkIn id v + checkSed id ["sed", "-e", v] = checkIn id v + checkSed _ _ = return () + + -- This should have used backreferences, but TDFA doesn't support them + sedRe = mkRegex "^s(.)([^\n]*)g?$" + isSimpleSed s = fromMaybe False $ do + [first,rest] <- matchRegex sedRe s + let delimiters = filter (== head first) rest + guard $ length delimiters == 2 + return True + checkIn id s = + when (isSimpleSed s) $ + style id 2001 "See if you can use ${variable//search/replace} instead." + prop_checkBraceExpansionVars1 = verify checkBraceExpansionVars "echo {1..$n}" prop_checkBraceExpansionVars2 = verifyNot checkBraceExpansionVars "echo {1,3,$n}" From 6debd59f025600a46d520bda2a3dc10b45c1b848 Mon Sep 17 00:00:00 2001 From: Gandalf- Date: Mon, 31 Dec 2018 18:41:47 -0800 Subject: [PATCH 047/763] Add context to case pattern warnings https://github.com/koalaman/shellcheck/issues/1039 --- src/ShellCheck/Analytics.hs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 21f429f..3882e0b 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2750,7 +2750,7 @@ prop_checkUnmatchableCases6 = verifyNot checkUnmatchableCases "case $f in ?*) tr prop_checkUnmatchableCases7 = verifyNot checkUnmatchableCases "case $f in $(x)) true;; asdf) false;; esac" prop_checkUnmatchableCases8 = verify checkUnmatchableCases "case $f in cow) true;; bar|cow) false;; esac" prop_checkUnmatchableCases9 = verifyNot checkUnmatchableCases "case $f in x) true;;& x) false;; esac" -checkUnmatchableCases _ t = +checkUnmatchableCases params t = case t of T_CaseExpression _ word list -> do -- Check all patterns for whether they can ever match @@ -2775,6 +2775,7 @@ checkUnmatchableCases _ t = where fst3 (x,_,_) = x snd3 (_,x,_) = x + tp = tokenPositions params check target candidate = potentially $ do candidateGlob <- wordToPseudoGlob candidate guard . not $ pseudoGlobsCanOverlap target candidateGlob @@ -2785,10 +2786,16 @@ checkUnmatchableCases _ t = checkDoms ((glob, Just x), rest) = case filter (\(_, p) -> x `pseudoGlobIsSuperSetof` p) valids of ((first,_):_) -> do - warn (getId glob) 2221 "This pattern always overrides a later one." - warn (getId first) 2222 "This pattern never matches because of a previous pattern." + warn (getId glob) 2221 $ "This pattern always overrides a later one" <> patternContext (getId first) + warn (getId first) 2222 $ "This pattern never matches because of a previous pattern" <> patternContext (getId glob) _ -> return () where + patternContext :: Id -> String + patternContext id = + case posLine . fst <$> Map.lookup id tp of + Just l -> " on line " <> show l <> "." + _ -> "." + valids = concatMap f rest f (x, Just y) = [(x,y)] f _ = [] From 97cb753d21e58a04dd457a2dc240b8e69acba5db Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 5 Jan 2019 11:36:42 -0800 Subject: [PATCH 048/763] Recognize --help (fixes #1441) --- shellcheck.hs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/shellcheck.hs b/shellcheck.hs index 6b9047c..84dee0a 100644 --- a/shellcheck.hs +++ b/shellcheck.hs @@ -102,10 +102,13 @@ options = [ (NoArg $ Flag "version" "true") "Print version information", Option "W" ["wiki-link-count"] (ReqArg (Flag "wiki-link-count") "NUM") - "The number of wiki links to show, when applicable.", + "The number of wiki links to show, when applicable", Option "x" ["external-sources"] - (NoArg $ Flag "externals" "true") "Allow 'source' outside of FILES" + (NoArg $ Flag "externals" "true") "Allow 'source' outside of FILES", + Option "" ["help"] + (NoArg $ Flag "help" "true") "Show this usage summary and exit" ] +getUsageInfo = usageInfo usageHeader options printErr = lift . hPutStrLn stderr @@ -114,7 +117,7 @@ parseArguments argv = case getOpt Permute options argv of (opts, files, []) -> return (opts, files) (_, _, errors) -> do - printErr $ concat errors ++ "\n" ++ usageInfo usageHeader options + printErr $ concat errors ++ "\n" ++ getUsageInfo throwError SyntaxFailure formats :: FormatterOptions -> Map.Map String (IO Formatter) @@ -271,6 +274,10 @@ parseOption flag options = liftIO printVersion throwError NoProblems + Flag "help" _ -> do + liftIO $ putStrLn getUsageInfo + throwError NoProblems + Flag "externals" _ -> return options { externalSources = True From 4a2b2c73968a7877c13c5aa4295b0f7dd0bc86e9 Mon Sep 17 00:00:00 2001 From: Gandalf- Date: Sun, 6 Jan 2019 17:42:17 -0800 Subject: [PATCH 049/763] Issue 1404 grep glob false positives https://github.com/koalaman/shellcheck/issues/1404 Some grep flags support globs; these are now all checked prevent false positives. --- src/ShellCheck/Checks/Commands.hs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index f4ead5b..e8ee853 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -208,6 +208,10 @@ prop_checkGrepRe12= verifyNot checkGrepRe "grep -F 'Foo*' file" prop_checkGrepRe13= verifyNot checkGrepRe "grep -- -foo bar*" prop_checkGrepRe14= verifyNot checkGrepRe "grep -e -foo bar*" prop_checkGrepRe15= verifyNot checkGrepRe "grep --regex -foo bar*" +prop_checkGrepRe16= verifyNot checkGrepRe "grep --include 'Foo*' file" +prop_checkGrepRe17= verifyNot checkGrepRe "grep --exclude 'Foo*' file" +prop_checkGrepRe18= verifyNot checkGrepRe "grep --exclude-dir 'Foo*' file" +prop_checkGrepRe19= verify checkGrepRe "grep -- 'Foo*' file" checkGrepRe = CommandCheck (Basename "grep") check where check cmd = f cmd (arguments cmd) @@ -230,7 +234,7 @@ checkGrepRe = CommandCheck (Basename "grep") check where when (isGlob re) $ warn (getId re) 2062 "Quote the grep pattern so the shell won't interpret it." - unless (cmd `hasFlag` "F") $ do + unless (any (hasFlag cmd) grepGlobFlags) $ do let string = concat $ oversimplify re if isConfusedGlobRegex string then warn (getId re) 2063 "Grep uses regex, but this looks like a glob." @@ -238,6 +242,8 @@ checkGrepRe = CommandCheck (Basename "grep") check where char <- getSuspiciousRegexWildcard string return $ info (getId re) 2022 $ "Note that unlike globs, " ++ [char] ++ "* here matches '" ++ [char, char, char] ++ "' but not '" ++ wordStartingWith char ++ "'." + where + grepGlobFlags = ["F", "include", "exclude", "exclude-dir"] wordStartingWith c = head . filter ([c] `isPrefixOf`) $ candidates From 394f4d6505755a2647086b0bb48271613518f199 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 7 Jan 2019 22:25:09 -0800 Subject: [PATCH 050/763] Make quicktest interpret test/shellcheck.hs directly --- quicktest | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/quicktest b/quicktest index 4f0702d..55041a7 100755 --- a/quicktest +++ b/quicktest @@ -4,15 +4,8 @@ # 'cabal test' remains the source of truth. ( - var=$(echo 'liftM and $ sequence [ - ShellCheck.Analytics.runTests - ,ShellCheck.Parser.runTests - ,ShellCheck.Checker.runTests - ,ShellCheck.Checks.Commands.runTests - ,ShellCheck.Checks.ShellSupport.runTests - ,ShellCheck.AnalyzerLib.runTests - ]' | tr -d '\n' | cabal repl ShellCheck 2>&1 | tee /dev/stderr) -if [[ $var == *$'\nTrue'* ]] +var=$(echo 'main' | ghci test/shellcheck.hs 2>&1 | tee /dev/stderr) +if [[ $var == *ExitSuccess* ]] then exit 0 else From ab2b0e11a300cb2cdcfb42cab632aed3e70d663b Mon Sep 17 00:00:00 2001 From: Tito Sacchi Date: Tue, 8 Jan 2019 20:20:26 +0100 Subject: [PATCH 051/763] Fix #1340 (SC2093 about removing "exec" should trigger in loops) --- src/ShellCheck/Analytics.hs | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 3882e0b..7310cd4 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1468,17 +1468,18 @@ prop_checkSpuriousExec5 = verifyNot checkSpuriousExec "exec > file; cmd" prop_checkSpuriousExec6 = verify checkSpuriousExec "exec foo > file; cmd" prop_checkSpuriousExec7 = verifyNot checkSpuriousExec "exec file; echo failed; exit 3" prop_checkSpuriousExec8 = verifyNot checkSpuriousExec "exec {origout}>&1- >tmp.log 2>&1; bar" +prop_checkSpuriousExec9 = verify checkSpuriousExec "for file in rc.d/*; do exec \"$file\"; done" checkSpuriousExec _ = doLists where - doLists (T_Script _ _ cmds) = doList cmds - doLists (T_BraceGroup _ cmds) = doList cmds - doLists (T_WhileExpression _ _ cmds) = doList cmds - doLists (T_UntilExpression _ _ cmds) = doList cmds - doLists (T_ForIn _ _ _ cmds) = doList cmds - doLists (T_ForArithmetic _ _ _ _ cmds) = doList cmds + doLists (T_Script _ _ cmds) = doList cmds False + doLists (T_BraceGroup _ cmds) = doList cmds False + doLists (T_WhileExpression _ _ cmds) = doList cmds True + doLists (T_UntilExpression _ _ cmds) = doList cmds True + doLists (T_ForIn _ _ _ cmds) = doList cmds True + doLists (T_ForArithmetic _ _ _ _ cmds) = doList cmds True doLists (T_IfExpression _ thens elses) = do - mapM_ (\(_, l) -> doList l) thens - doList elses + mapM_ (\(_, l) -> doList l False) thens + doList elses False doLists _ = return () stripCleanup = reverse . dropWhile cleanup . reverse @@ -1487,10 +1488,15 @@ checkSpuriousExec _ = doLists cleanup _ = False doList = doList' . stripCleanup - doList' t@(current:following:_) = do + -- The second parameter is True if we are in a loop + -- In that case we should emit the warning also if `exec' is the last statement + doList' t@(current:following:_) False = do commentIfExec current - doList (tail t) - doList' _ = return () + doList (tail t) False + doList' (current:tail) True = do + commentIfExec current + doList tail True + doList' _ _ = return () commentIfExec (T_Pipeline id _ list) = mapM_ commentIfExec $ take 1 list From 434b9047462f2fe8472abae0d698abf92b16fa78 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 8 Jan 2019 17:22:39 -0800 Subject: [PATCH 052/763] Process replacements according to AST depth (fixes #1431) --- ShellCheck.cabal | 5 +- src/ShellCheck/Analytics.hs | 15 +- src/ShellCheck/Fixer.hs | 341 +++++++++++++++++++++++++---- src/ShellCheck/Formatter/Format.hs | 10 +- src/ShellCheck/Formatter/TTY.hs | 61 +++--- src/ShellCheck/Interface.hs | 16 +- test/shellcheck.hs | 18 +- 7 files changed, 382 insertions(+), 84 deletions(-) diff --git a/ShellCheck.cabal b/ShellCheck.cabal index adf5c7d..3345e32 100644 --- a/ShellCheck.cabal +++ b/ShellCheck.cabal @@ -49,9 +49,10 @@ library build-depends: semigroups build-depends: + aeson, + array, -- GHC 7.6.3 (base 4.6.0.1) is buggy (#1131, #1119) in optimized mode. -- Just disable that version entirely to fail fast. - aeson, base > 4.6.0.1 && < 5, bytestring, containers >= 0.5, @@ -91,6 +92,7 @@ executable shellcheck semigroups build-depends: aeson, + array, base >= 4 && < 5, bytestring, deepseq >= 1.4.0.0, @@ -107,6 +109,7 @@ test-suite test-shellcheck type: exitcode-stdio-1.0 build-depends: aeson, + array, base >= 4 && < 5, bytestring, deepseq >= 1.4.0.0, diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 3882e0b..caac88f 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -250,11 +250,14 @@ replaceStart id params n r = new_end = start { posColumn = posColumn start + n } + depth = length $ getPath (parentMap params) (T_EOF id) in newReplacement { repStartPos = start, repEndPos = new_end, - repString = r + repString = r, + repPrecedence = depth, + repInsertionPoint = InsertAfter } replaceEnd id params n r = let tp = tokenPositions params @@ -265,11 +268,14 @@ replaceEnd id params n r = new_end = end { posColumn = posColumn end } + depth = length $ getPath (parentMap params) (T_EOF id) in newReplacement { repStartPos = new_start, repEndPos = new_end, - repString = r + repString = r, + repPrecedence = depth, + repInsertionPoint = InsertBefore } surroundWidth id params s = fixWith [replaceStart id params 0 s, replaceEnd id params 0 s] fixWith fixes = newFix { fixReplacements = fixes } @@ -1676,9 +1682,8 @@ checkSpacefulness params t = "This default assignment may cause DoS due to globbing. Quote it." else makeCommentWithFix InfoC (getId token) 2086 - "Double quote to prevent globbing and word splitting." (surroundWidth (getId token) params "\"") - -- makeComment InfoC (getId token) 2086 - -- "Double quote to prevent globbing and word splitting." + "Double quote to prevent globbing and word splitting." + (surroundWidth (getId token) params "\"") writeF _ _ name (DataString SourceExternal) = setSpaces name True >> return [] writeF _ _ name (DataString SourceInteger) = setSpaces name False >> return [] diff --git a/src/ShellCheck/Fixer.hs b/src/ShellCheck/Fixer.hs index 9c4837c..c1f52a7 100644 --- a/src/ShellCheck/Fixer.hs +++ b/src/ShellCheck/Fixer.hs @@ -1,8 +1,33 @@ -module ShellCheck.Fixer (applyFix , replaceMultiLines, Ranged(..)) where +{- + Copyright 2018-2019 Vidar Holen, Ng Zhi An + + This file is part of ShellCheck. + https://www.shellcheck.net + + ShellCheck is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + ShellCheck is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +-} + +{-# LANGUAGE TemplateHaskell #-} +module ShellCheck.Fixer (applyFix, mapPositions, Ranged(..), runTests) where import ShellCheck.Interface +import Control.Monad.State +import Data.Array import Data.List import Data.Semigroup +import GHC.Exts (sortWith) +import Test.QuickCheck -- The Ranged class is used for types that has a start and end position. class Ranged a where @@ -19,6 +44,27 @@ class Ranged a where -- Set a new start and end position on a Ranged setRange :: (Position, Position) -> a -> a +-- Tests auto-verify that overlap commutes +assertOverlap x y = overlap x y && overlap y x +assertNoOverlap x y = not (overlap x y) && not (overlap y x) + +prop_overlap_contiguous = assertNoOverlap + (tFromStart 10 12 "foo" 1) + (tFromStart 12 14 "bar" 2) + +prop_overlap_adjacent_zerowidth = assertNoOverlap + (tFromStart 3 3 "foo" 1) + (tFromStart 3 3 "bar" 2) + +prop_overlap_enclosed = assertOverlap + (tFromStart 3 5 "foo" 1) + (tFromStart 1 10 "bar" 2) + +prop_overlap_partial = assertOverlap + (tFromStart 1 5 "foo" 1) + (tFromStart 3 7 "bar" 2) + + instance Ranged PositionedComment where start = pcStartPos end = pcEndPos @@ -35,44 +81,60 @@ instance Ranged Replacement where repEndPos = e } -instance Ranged a => Ranged [a] where - start [] = newPosition - start xs = (minimum . map start) xs - end [] = newPosition - end xs = (maximum . map end) xs - setRange (s, e) rs = map (setRange (s, e)) rs - -instance Ranged Fix where - start = start . fixReplacements - end = end . fixReplacements - setRange (s, e) f = f { - fixReplacements = setRange (s, e) (fixReplacements f) - } - --- The Monoid instance for Fix merges replacements that do not overlap. +-- The Monoid instance for Fix merges fixes that do not conflict. +-- TODO: Make an efficient 'mconcat' instance Monoid Fix where mempty = newFix mappend = (<>) instance Semigroup Fix where - f1 <> f2 = if overlap f1 f2 then f1 else newFix { + f1 <> f2 = + -- FIXME: This might need to also discard adjacent zero-width ranges for + -- when two fixes change the same AST node, e.g. `foo` -> "$(foo)" + if or [ r2 `overlap` r1 | r1 <- fixReplacements f1, r2 <- fixReplacements f2 ] + then f1 + else newFix { fixReplacements = fixReplacements f1 ++ fixReplacements f2 - } + } +mapPositions :: (Position -> Position) -> Fix -> Fix +mapPositions f = adjustFix + where + adjustReplacement rep = + rep { + repStartPos = f $ repStartPos rep, + repEndPos = f $ repEndPos rep + } + adjustFix fix = + fix { + fixReplacements = map adjustReplacement $ fixReplacements fix + } + +multiToSingleLine :: [Fix] -> Array Int String -> ([Fix], String) +multiToSingleLine fixes lines = + (map (mapPositions adjust) fixes, unlines $ elems lines) + where + -- A prefix sum tree from line number to column shift. + -- FIXME: The tree will be totally unbalanced. + shiftTree :: PSTree Int + shiftTree = + foldl (\t (n,s) -> addPSValue (n+1) (length s + 1) t) newPSTree $ + assocs lines + singleString = unlines $ elems lines + adjust pos = + pos { + posLine = 1, + posColumn = (posColumn pos) + + (fromIntegral $ getPrefixSum (fromIntegral $ posLine pos) shiftTree) + } + +-- Apply a fix and return resulting lines. +-- The number of lines can increase or decrease with no obvious mapping back, so +-- the function does not return an array. +applyFix :: Fix -> Array Int String -> [String] applyFix fix fileLines = - -- apply replacements in sorted order by end position - let sorted = (removeOverlap . reverse . sort) (fixReplacements fix) in - applyReplacement sorted fileLines - where - applyReplacement [] s = s - applyReplacement (rep:xs) s = applyReplacement xs $ replaceMultiLines rep s - -- prereq: list is already sorted by start position - removeOverlap [] = [] - removeOverlap (x:xs) = checkoverlap x xs - checkoverlap x [] = x:[] - checkoverlap x (y:ys) = - if overlap x y then x:(removeOverlap ys) else x:y:(removeOverlap ys) - + let (adjustedFixes, singleLine) = multiToSingleLine [fix] fileLines + in lines . runFixer $ applyFixes2 adjustedFixes singleLine -- A replacement that spans multiple line is applied by: -- 1. merging the affected lines into a single string using `unlines` @@ -111,14 +173,13 @@ replaceMultiLines rep fileLines = -- this can replace doReplace in xs ++ replacedLines ++ zs --- FIXME: Work correctly with tabs -- start and end comes from pos, which is 1 based --- doReplace 0 0 "1234" "A" -> "A1234" -- technically not valid --- doReplace 1 1 "1234" "A" -> "A1234" --- doReplace 1 2 "1234" "A" -> "A234" --- doReplace 3 3 "1234" "A" -> "12A34" --- doReplace 4 4 "1234" "A" -> "123A4" --- doReplace 5 5 "1234" "A" -> "1234A" +prop_doReplace1 = doReplace 0 0 "1234" "A" == "A1234" -- technically not valid +prop_doReplace2 = doReplace 1 1 "1234" "A" == "A1234" +prop_doReplace3 = doReplace 1 2 "1234" "A" == "A234" +prop_doReplace4 = doReplace 3 3 "1234" "A" == "12A34" +prop_doReplace5 = doReplace 4 4 "1234" "A" == "123A4" +prop_doReplace6 = doReplace 5 5 "1234" "A" == "1234A" doReplace start end o r = let si = fromIntegral (start-1) ei = fromIntegral (end-1) @@ -126,3 +187,207 @@ doReplace start end o r = (y, z) = splitAt (ei - si) xs in x ++ r ++ z + +-- Fail if the 'expected' string is not result when applying 'fixes' to 'original'. +testFixes :: String -> String -> [Fix] -> Bool +testFixes expected original fixes = + actual == expected + where + actual = runFixer (applyFixes2 fixes original) + + +-- A Fixer allows doing repeated modifications of a string where each +-- replacement automatically accounts for shifts from previous ones. +type Fixer a = State (PSTree Int) a + +-- Apply a single replacement using its indices into the original string. +-- It does not handle multiple lines, all line indices must be 1. +applyReplacement2 :: Replacement -> String -> Fixer String +applyReplacement2 rep string = do + tree <- get + let transform pos = pos + getPrefixSum pos tree + let originalPos = (repStartPos rep, repEndPos rep) + (oldStart, oldEnd) = tmap (fromInteger . posColumn) originalPos + (newStart, newEnd) = tmap transform (oldStart, oldEnd) + + let (l1, l2) = tmap posLine originalPos in + when (l1 /= 1 || l2 /= 1) $ + error "ShellCheck internal error, please report: bad cross-line fix" + + let replacer = repString rep + let shift = (length replacer) - (oldEnd - oldStart) + let insertionPoint = + case repInsertionPoint rep of + InsertBefore -> oldStart + InsertAfter -> oldEnd+1 + put $ addPSValue insertionPoint shift tree + + return $ doReplace newStart newEnd string replacer + where + tmap f (a,b) = (f a, f b) + +-- Apply a list of Replacements in the correct order +applyReplacements2 :: [Replacement] -> String -> Fixer String +applyReplacements2 reps str = + foldM (flip applyReplacement2) str $ + reverse $ sortWith repPrecedence reps + +-- Apply all fixes with replacements in the correct order +applyFixes2 :: [Fix] -> String -> Fixer String +applyFixes2 fixes = applyReplacements2 (concatMap fixReplacements fixes) + +-- Get the final value of a Fixer. +runFixer :: Fixer a -> a +runFixer f = evalState f newPSTree + + + +-- A Prefix Sum Tree that lets you look up the sum of values at and below an index. +-- It's implemented essentially as a Fenwick tree without the bit-based balancing. +-- The last Num is the sum of the left branch plus current element. +data PSTree n = PSBranch n (PSTree n) (PSTree n) n | PSLeaf + deriving (Show) + +newPSTree :: Num n => PSTree n +newPSTree = PSLeaf + +-- Get the sum of values whose keys are <= 'target' +getPrefixSum :: (Ord n, Num n) => n -> PSTree n -> n +getPrefixSum = f 0 + where + f sum _ PSLeaf = sum + f sum target (PSBranch pivot left right cumulative) = + case () of + _ | target < pivot -> f sum target left + _ | target > pivot -> f (sum+cumulative) target right + _ -> sum+cumulative + +-- Add a value to the Prefix Sum tree at the given index. +-- Values accumulate: addPSValue 42 2 . addPSValue 42 3 == addPSValue 42 5 +addPSValue :: (Ord n, Num n) => n -> n -> PSTree n -> PSTree n +addPSValue key value tree = if value == 0 then tree else f tree + where + f PSLeaf = PSBranch key PSLeaf PSLeaf value + f (PSBranch pivot left right sum) = + case () of + _ | key < pivot -> PSBranch pivot (f left) right (sum + value) + _ | key > pivot -> PSBranch pivot left (f right) sum + _ -> PSBranch pivot left right (sum + value) + +prop_pstreeSumsCorrectly kvs targets = + let + -- Trivial O(n * m) implementation + dumbPrefixSums :: [(Int, Int)] -> [Int] -> [Int] + dumbPrefixSums kvs targets = + let prefixSum target = sum . map snd . filter (\(k,v) -> k <= target) $ kvs + in map prefixSum targets + -- PSTree O(n * log m) implementation + smartPrefixSums :: [(Int, Int)] -> [Int] -> [Int] + smartPrefixSums kvs targets = + let tree = foldl (\tree (pos, shift) -> addPSValue pos shift tree) PSLeaf kvs + in map (\x -> getPrefixSum x tree) targets + in smartPrefixSums kvs targets == dumbPrefixSums kvs targets + + +-- Semi-convenient functions for constructing tests. +testFix :: [Replacement] -> Fix +testFix list = newFix { + fixReplacements = list + } + +tFromStart :: Int -> Int -> String -> Int -> Replacement +tFromStart start end repl order = + newReplacement { + repStartPos = newPosition { + posLine = 1, + posColumn = fromIntegral start + }, + repEndPos = newPosition { + posLine = 1, + posColumn = fromIntegral end + }, + repString = repl, + repPrecedence = order, + repInsertionPoint = InsertAfter + } + +tFromEnd start end repl order = + (tFromStart start end repl order) { + repInsertionPoint = InsertBefore + } + +prop_simpleFix1 = testFixes "hello world" "hell world" [ + testFix [ + tFromEnd 5 5 "o" 1 + ]] + +prop_anchorsLeft = testFixes "-->foobar<--" "--><--" [ + testFix [ + tFromStart 4 4 "foo" 1, + tFromStart 4 4 "bar" 2 + ]] + +prop_anchorsRight = testFixes "-->foobar<--" "--><--" [ + testFix [ + tFromEnd 4 4 "bar" 1, + tFromEnd 4 4 "foo" 2 + ]] + +prop_anchorsBoth1 = testFixes "-->foobar<--" "--><--" [ + testFix [ + tFromStart 4 4 "bar" 2, + tFromEnd 4 4 "foo" 1 + ]] + +prop_anchorsBoth2 = testFixes "-->foobar<--" "--><--" [ + testFix [ + tFromEnd 4 4 "foo" 2, + tFromStart 4 4 "bar" 1 + ]] + +prop_composeFixes1 = testFixes "cd \"$1\" || exit" "cd $1" [ + testFix [ + tFromStart 4 4 "\"" 10, + tFromEnd 6 6 "\"" 10 + ], + testFix [ + tFromEnd 6 6 " || exit" 5 + ]] + +prop_composeFixes2 = testFixes "$(\"$1\")" "`$1`" [ + testFix [ + tFromStart 1 2 "$(" 5, + tFromEnd 4 5 ")" 5 + ], + testFix [ + tFromStart 2 2 "\"" 10, + tFromEnd 4 4 "\"" 10 + ]] + +prop_composeFixes3 = testFixes "(x)[x]" "xx" [ + testFix [ + tFromStart 1 1 "(" 4, + tFromEnd 2 2 ")" 3, + tFromStart 2 2 "[" 2, + tFromEnd 3 3 "]" 1 + ]] + +prop_composeFixes4 = testFixes "(x)[x]" "xx" [ + testFix [ + tFromStart 1 1 "(" 4, + tFromStart 2 2 "[" 3, + tFromEnd 2 2 ")" 2, + tFromEnd 3 3 "]" 1 + ]] + +prop_composeFixes5 = testFixes "\"$(x)\"" "`x`" [ + testFix [ + tFromStart 1 2 "$(" 2, + tFromEnd 3 4 ")" 2, + tFromStart 1 1 "\"" 1, + tFromEnd 4 4 "\"" 1 + ]] + + +return [] +runTests = $quickCheckAll diff --git a/src/ShellCheck/Formatter/Format.hs b/src/ShellCheck/Formatter/Format.hs index 5c3c444..1e2b57f 100644 --- a/src/ShellCheck/Formatter/Format.hs +++ b/src/ShellCheck/Formatter/Format.hs @@ -22,6 +22,7 @@ module ShellCheck.Formatter.Format where import ShellCheck.Data import ShellCheck.Interface import ShellCheck.Fixer +import Data.Array -- A formatter that carries along an arbitrary piece of data data Formatter = Formatter { @@ -51,11 +52,12 @@ severityText pc = makeNonVirtual comments contents = map fix comments where - ls = lines contents - fix c = realign c ls + list = lines contents + arr = listArray (1, length list) list + fix c = realign c arr -- Realign a Ranged from a tabstop of 8 to 1 -realign :: Ranged a => a -> [String] -> a +realign :: Ranged a => a -> Array Int String -> a realign range ls = let startColumn = realignColumn lineNo colNo range endColumn = realignColumn endLineNo endColNo range @@ -65,7 +67,7 @@ realign range ls = where realignColumn lineNo colNo c = if lineNo c > 0 && lineNo c <= fromIntegral (length ls) - then real (ls !! fromIntegral (lineNo c - 1)) 0 0 (colNo c) + then real (ls ! fromIntegral (lineNo c)) 0 0 (colNo c) else colNo c real _ r v target | target <= v = r -- hit this case at the end of line, and if we don't hit the target diff --git a/src/ShellCheck/Formatter/TTY.hs b/src/ShellCheck/Formatter/TTY.hs index ffc5e93..254287f 100644 --- a/src/ShellCheck/Formatter/TTY.hs +++ b/src/ShellCheck/Formatter/TTY.hs @@ -24,6 +24,7 @@ import ShellCheck.Interface import ShellCheck.Formatter.Format import Control.Monad +import Data.Array import Data.Foldable import Data.Ord import Data.IORef @@ -37,6 +38,8 @@ wikiLink = "https://www.shellcheck.net/wiki/" -- An arbitrary Ord thing to order warnings type Ranking = (Char, Severity, Integer) +-- Ansi coloring function +type ColorFunc = (String -> String -> String) format :: FormatterOptions -> IO Formatter format options = do @@ -119,59 +122,66 @@ outputForFile color sys comments = do let fileName = sourceFile (head comments) result <- (siReadFile sys) fileName let contents = either (const "") id result - let fileLines = lines contents - let lineCount = fromIntegral $ length fileLines + let fileLinesList = lines contents + let lineCount = length fileLinesList + let fileLines = listArray (1, lineCount) fileLinesList let groups = groupWith lineNo comments mapM_ (\commentsForLine -> do - let lineNum = lineNo (head commentsForLine) + let lineNum = fromIntegral $ lineNo (head commentsForLine) let line = if lineNum < 1 || lineNum > lineCount then "" - else fileLines !! fromIntegral (lineNum - 1) + else fileLines ! fromIntegral lineNum putStrLn "" putStrLn $ color "message" $ "In " ++ fileName ++" line " ++ show lineNum ++ ":" putStrLn (color "source" line) mapM_ (\c -> putStrLn (color (severityText c) $ cuteIndent c)) commentsForLine putStrLn "" - showFixedString color comments lineNum fileLines + showFixedString color commentsForLine (fromIntegral lineNum) fileLines ) groups -hasApplicableFix lineNum comment = fromMaybe False $ do - replacements <- fixReplacements <$> pcFix comment - guard $ all (\c -> onSameLine (repStartPos c) && onSameLine (repEndPos c)) replacements - return True +-- Pick out only the lines necessary to show a fix in action +sliceFile :: Fix -> Array Int String -> (Fix, Array Int String) +sliceFile fix lines = + (mapPositions adjust fix, sliceLines lines) where - onSameLine pos = posLine pos == lineNum + (minLine, maxLine) = + foldl (\(mm, mx) pos -> ((min mm $ fromIntegral $ posLine pos), (max mx $ fromIntegral $ posLine pos))) + (maxBound, minBound) $ + concatMap (\x -> [repStartPos x, repEndPos x]) $ fixReplacements fix + sliceLines :: Array Int String -> Array Int String + sliceLines = ixmap (1, maxLine - minLine + 1) (\x -> x + minLine - 1) + adjust pos = + pos { + posLine = posLine pos - (fromIntegral minLine) + 1 + } --- FIXME: Work correctly with multiple replacements +showFixedString :: ColorFunc -> [PositionedComment] -> Int -> Array Int String -> IO () showFixedString color comments lineNum fileLines = - let line = fileLines !! fromIntegral (lineNum - 1) in - -- need to check overlaps - case filter (hasApplicableFix lineNum) comments of + let line = fileLines ! fromIntegral lineNum in + case mapMaybe pcFix comments of [] -> return () - -- all the fixes are single-line only, but there could be multiple - -- fixes for that single line. We can fold the fixes (which removes - -- overlaps), and apply it as a single fix with multiple replacements. - applicableComments -> do - let mergedFix = (realignFix . fold . catMaybes . (map pcFix)) applicableComments + fixes -> do + -- Folding automatically removes overlap + let mergedFix = realignFix $ fold fixes + -- We show the complete, associated fixes, whether or not it includes this and/or unrelated lines. + let (excerptFix, excerpt) = sliceFile mergedFix fileLines -- in the spirit of error prone putStrLn $ color "message" "Did you mean: " - putStrLn $ unlines $ fixedString mergedFix fileLines + putStrLn $ unlines $ fixedString excerptFix excerpt where + -- FIXME: This should be handled by Fixer realignFix f = f { fixReplacements = map fix (fixReplacements f) } fix r = realign r fileLines -fixedString :: Fix -> [String] -> [String] +fixedString :: Fix -> Array Int String -> [String] fixedString fix fileLines = case (fixReplacements fix) of [] -> [] reps -> -- applyReplacement returns the full update file, we really only care about the changed lines -- so we calculate overlapping lines using replacements - drop start $ take end $ applyFix fix fileLines - where - start = (fromIntegral $ minimum $ map (posLine . repStartPos) reps) - 1 - end = fromIntegral $ maximum $ map (posLine . repEndPos) reps + applyFix fix fileLines cuteIndent :: PositionedComment -> String cuteIndent comment = @@ -187,6 +197,7 @@ cuteIndent comment = code num = "SC" ++ show num +getColorFunc :: ColorOption -> IO ColorFunc getColorFunc colorOption = do term <- hIsTerminalDevice stdout let windows = "mingw" `isPrefixOf` os diff --git a/src/ShellCheck/Interface.hs b/src/ShellCheck/Interface.hs index ea70c15..dd53f6f 100644 --- a/src/ShellCheck/Interface.hs +++ b/src/ShellCheck/Interface.hs @@ -52,7 +52,8 @@ module ShellCheck.Interface , newComment , Fix(fixReplacements) , newFix - , Replacement(repStartPos, repEndPos, repString) + , InsertionPoint(InsertBefore, InsertAfter) + , Replacement(repStartPos, repEndPos, repString, repPrecedence, repInsertionPoint) , newReplacement ) where @@ -209,16 +210,25 @@ newComment = Comment { data Replacement = Replacement { repStartPos :: Position, repEndPos :: Position, - repString :: String + repString :: String, + -- Order in which the replacements should happen: highest precedence first. + repPrecedence :: Int, + -- Whether to insert immediately before or immediately after the specified region. + repInsertionPoint :: InsertionPoint } deriving (Show, Eq, Generic, NFData) +data InsertionPoint = InsertBefore | InsertAfter + deriving (Show, Eq, Generic, NFData) + instance Ord Replacement where compare r1 r2 = (repStartPos r1) `compare` (repStartPos r2) newReplacement = Replacement { repStartPos = newPosition, repEndPos = newPosition, - repString = "" + repString = "", + repPrecedence = 1, + repInsertionPoint = InsertAfter } data Fix = Fix { diff --git a/test/shellcheck.hs b/test/shellcheck.hs index 6106d9a..8f858d6 100644 --- a/test/shellcheck.hs +++ b/test/shellcheck.hs @@ -2,22 +2,24 @@ module Main where import Control.Monad import System.Exit -import qualified ShellCheck.Checker import qualified ShellCheck.Analytics import qualified ShellCheck.AnalyzerLib -import qualified ShellCheck.Parser +import qualified ShellCheck.Checker import qualified ShellCheck.Checks.Commands import qualified ShellCheck.Checks.ShellSupport +import qualified ShellCheck.Fixer +import qualified ShellCheck.Parser main = do putStrLn "Running ShellCheck tests..." results <- sequence [ - ShellCheck.Checker.runTests, - ShellCheck.Checks.Commands.runTests, - ShellCheck.Checks.ShellSupport.runTests, - ShellCheck.Analytics.runTests, - ShellCheck.AnalyzerLib.runTests, - ShellCheck.Parser.runTests + ShellCheck.Analytics.runTests + ,ShellCheck.AnalyzerLib.runTests + ,ShellCheck.Checker.runTests + ,ShellCheck.Checks.Commands.runTests + ,ShellCheck.Checks.ShellSupport.runTests + ,ShellCheck.Fixer.runTests + ,ShellCheck.Parser.runTests ] if and results then exitSuccess From baa4d2e555a7557315e788498047bbef7de328ef Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 8 Jan 2019 19:06:00 -0800 Subject: [PATCH 053/763] Let checkGrepRe only parse flags once --- src/ShellCheck/Checks/Commands.hs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index e8ee853..93720e8 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -234,7 +234,7 @@ checkGrepRe = CommandCheck (Basename "grep") check where when (isGlob re) $ warn (getId re) 2062 "Quote the grep pattern so the shell won't interpret it." - unless (any (hasFlag cmd) grepGlobFlags) $ do + unless (any (`elem` flags) grepGlobFlags) $ do let string = concat $ oversimplify re if isConfusedGlobRegex string then warn (getId re) 2063 "Grep uses regex, but this looks like a glob." @@ -243,6 +243,7 @@ checkGrepRe = CommandCheck (Basename "grep") check where return $ info (getId re) 2022 $ "Note that unlike globs, " ++ [char] ++ "* here matches '" ++ [char, char, char] ++ "' but not '" ++ wordStartingWith char ++ "'." where + flags = map snd $ getAllFlags cmd grepGlobFlags = ["F", "include", "exclude", "exclude-dir"] wordStartingWith c = From 263401cfcbccb3ce9c294b5a1ec1c53ab79f18c7 Mon Sep 17 00:00:00 2001 From: Gandalf- Date: Sun, 6 Jan 2019 18:52:17 -0800 Subject: [PATCH 054/763] Issue 1318 single comma array delimiter Issue https://github.com/koalaman/shellcheck/issues/1318 The case in which a single comma, with no spaces, used in an array assignment is now caught for SC2054. --- src/ShellCheck/Analytics.hs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 3882e0b..0e71152 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1245,6 +1245,7 @@ prop_checkCommarrays3 = verifyNot checkCommarrays "cow=(1 \"foo,bar\" 3)" prop_checkCommarrays4 = verifyNot checkCommarrays "cow=('one,' 'two')" prop_checkCommarrays5 = verify checkCommarrays "a=([a]=b, [c]=d)" prop_checkCommarrays6 = verify checkCommarrays "a=([a]=b,[c]=d,[e]=f)" +prop_checkCommarrays7 = verify checkCommarrays "a=(1,2)" checkCommarrays _ (T_Array id l) = when (any (isCommaSeparated . literal) l) $ warn id 2054 "Use spaces, not commas, to separate array elements." @@ -1252,9 +1253,9 @@ checkCommarrays _ (T_Array id l) = literal (T_IndexedElement _ _ l) = literal l literal (T_NormalWord _ l) = concatMap literal l literal (T_Literal _ str) = str - literal _ = "str" + literal _ = "" - isCommaSeparated str = "," `isSuffixOf` str || length (filter (== ',') str) > 1 + isCommaSeparated = elem ',' checkCommarrays _ _ = return () prop_checkOrNeq1 = verify checkOrNeq "if [[ $lol -ne cow || $lol -ne foo ]]; then echo foo; fi" From df7f00eaedf88eb5d1d99169e261505d339e3a1b Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 8 Jan 2019 22:16:17 -0800 Subject: [PATCH 055/763] Remove duplicate `pathTo` and unused `replaceMultiLines` --- src/ShellCheck/AnalyzerLib.hs | 4 --- src/ShellCheck/Checks/Commands.hs | 2 +- src/ShellCheck/Fixer.hs | 57 ++++++++++++------------------- 3 files changed, 22 insertions(+), 41 deletions(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 90a79e0..37a96ad 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -380,10 +380,6 @@ isParentOf tree parent child = parents params = getPath (parentMap params) -pathTo t = do - parents <- reader parentMap - return $ getPath parents t - -- Find the first match in a list where the predicate is Just True. -- Stops if it's Just False and ignores Nothing. findFirst :: (a -> Maybe Bool) -> [a] -> Maybe a diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 93720e8..6c82a4f 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -484,7 +484,7 @@ prop_checkInteractiveSu4 = verifyNot checkInteractiveSu "su root < script" checkInteractiveSu = CommandCheck (Basename "su") f where f cmd = when (length (arguments cmd) <= 1) $ do - path <- pathTo cmd + path <- getPathM cmd when (all undirected path) $ info (getId cmd) 2117 "To run commands as another user, use su -c or sudo." diff --git a/src/ShellCheck/Fixer.hs b/src/ShellCheck/Fixer.hs index c1f52a7..22b528b 100644 --- a/src/ShellCheck/Fixer.hs +++ b/src/ShellCheck/Fixer.hs @@ -97,6 +97,7 @@ instance Semigroup Fix where fixReplacements = fixReplacements f1 ++ fixReplacements f2 } +-- Conveniently apply a transformation to positions in a Fix mapPositions :: (Position -> Position) -> Fix -> Fix mapPositions f = adjustFix where @@ -110,6 +111,26 @@ mapPositions f = adjustFix fixReplacements = map adjustReplacement $ fixReplacements fix } + +-- A replacement that spans multiple line is applied by: +-- 1. merging the affected lines into a single string using `unlines` +-- 2. apply the replacement as if it only spanned a single line +-- The tricky part is adjusting the end column of the replacement +-- (the end line doesn't matter because there is only one line) +-- +-- aaS <--- start of replacement (row 1 column 3) +-- bbbb +-- cEc +-- \------- end of replacement (row 3 column 2) +-- +-- a flattened string will look like: +-- +-- "aaS\nbbbb\ncEc\n" +-- +-- The column of E has to be adjusted by: +-- 1. lengths of lines to be replaced, except the end row itself +-- 2. end column of the replacement +-- 3. number of '\n' by `unlines` multiToSingleLine :: [Fix] -> Array Int String -> ([Fix], String) multiToSingleLine fixes lines = (map (mapPositions adjust) fixes, unlines $ elems lines) @@ -136,42 +157,6 @@ applyFix fix fileLines = let (adjustedFixes, singleLine) = multiToSingleLine [fix] fileLines in lines . runFixer $ applyFixes2 adjustedFixes singleLine --- A replacement that spans multiple line is applied by: --- 1. merging the affected lines into a single string using `unlines` --- 2. apply the replacement as if it only spanned a single line --- The tricky part is adjusting the end column of the replacement --- (the end line doesn't matter because there is only one line) --- --- aaS <--- start of replacement (row 1 column 3) --- bbbb --- cEc --- \------- end of replacement (row 3 column 2) --- --- a flattened string will look like: --- --- "aaS\nbbbb\ncEc\n" --- --- The column of E has to be adjusted by: --- 1. lengths of lines to be replaced, except the end row itself --- 2. end column of the replacement --- 3. number of '\n' by `unlines` --- Returns the original lines from the file with the replacement applied. --- Multiline replacements completely overwrite new lines in the original string. --- e.g. if the replacement spans 2 lines, but the replacement string does not --- have a '\n', then the number of replaced lines will be 1 shorter. -replaceMultiLines rep fileLines = -- this can replace doReplace - let startRow = fromIntegral $ (posLine . repStartPos) rep - endRow = fromIntegral $ (posLine . repEndPos) rep - (ys, zs) = splitAt endRow fileLines - (xs, toReplaceLines) = splitAt (startRow-1) ys - lengths = fromIntegral $ sum (map length (init toReplaceLines)) - newlines = fromIntegral $ (length toReplaceLines - 1) -- for the '\n' from unlines - original = unlines toReplaceLines - startCol = ((posColumn . repStartPos) rep) - endCol = ((posColumn . repEndPos) rep + newlines + lengths) - replacedLines = (lines $ doReplace startCol endCol original (repString rep)) - in - xs ++ replacedLines ++ zs -- start and end comes from pos, which is 1 based prop_doReplace1 = doReplace 0 0 "1234" "A" == "A1234" -- technically not valid From fd2beaadfad5f01fade9cdb344bd3fe750f8e509 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 9 Jan 2019 18:08:59 -0800 Subject: [PATCH 056/763] Make Fixer responsible for realigning tab stops --- src/ShellCheck/Fixer.hs | 37 +++++++++++++++++++++++++++--- src/ShellCheck/Formatter/Format.hs | 26 +-------------------- src/ShellCheck/Formatter/TTY.hs | 20 ++++------------ 3 files changed, 39 insertions(+), 44 deletions(-) diff --git a/src/ShellCheck/Fixer.hs b/src/ShellCheck/Fixer.hs index 22b528b..12de3c2 100644 --- a/src/ShellCheck/Fixer.hs +++ b/src/ShellCheck/Fixer.hs @@ -19,7 +19,7 @@ -} {-# LANGUAGE TemplateHaskell #-} -module ShellCheck.Fixer (applyFix, mapPositions, Ranged(..), runTests) where +module ShellCheck.Fixer (applyFix, removeTabStops, mapPositions, Ranged(..), runTests) where import ShellCheck.Interface import Control.Monad.State @@ -111,6 +111,30 @@ mapPositions f = adjustFix fixReplacements = map adjustReplacement $ fixReplacements fix } +-- Rewrite a Ranged from a tabstop of 8 to 1 +removeTabStops :: Ranged a => a -> Array Int String -> a +removeTabStops range ls = + let startColumn = realignColumn lineNo colNo range + endColumn = realignColumn endLineNo endColNo range + startPosition = (start range) { posColumn = startColumn } + endPosition = (end range) { posColumn = endColumn } in + setRange (startPosition, endPosition) range + where + realignColumn lineNo colNo c = + if lineNo c > 0 && lineNo c <= fromIntegral (length ls) + then real (ls ! fromIntegral (lineNo c)) 0 0 (colNo c) + else colNo c + real _ r v target | target <= v = r + -- hit this case at the end of line, and if we don't hit the target + -- return real + (target - v) + real [] r v target = r + (target - v) + real ('\t':rest) r v target = real rest (r+1) (v + 8 - (v `mod` 8)) target + real (_:rest) r v target = real rest (r+1) (v+1) target + lineNo = posLine . start + endLineNo = posLine . end + colNo = posColumn . start + endColNo = posColumn . end + -- A replacement that spans multiple line is applied by: -- 1. merging the affected lines into a single string using `unlines` @@ -154,8 +178,15 @@ multiToSingleLine fixes lines = -- the function does not return an array. applyFix :: Fix -> Array Int String -> [String] applyFix fix fileLines = - let (adjustedFixes, singleLine) = multiToSingleLine [fix] fileLines - in lines . runFixer $ applyFixes2 adjustedFixes singleLine + let + untabbed = fix { + fixReplacements = + map (\c -> removeTabStops c fileLines) $ + fixReplacements fix + } + (adjustedFixes, singleLine) = multiToSingleLine [untabbed] fileLines + in + lines . runFixer $ applyFixes2 adjustedFixes singleLine -- start and end comes from pos, which is 1 based diff --git a/src/ShellCheck/Formatter/Format.hs b/src/ShellCheck/Formatter/Format.hs index 1e2b57f..57b9d71 100644 --- a/src/ShellCheck/Formatter/Format.hs +++ b/src/ShellCheck/Formatter/Format.hs @@ -54,29 +54,5 @@ makeNonVirtual comments contents = where list = lines contents arr = listArray (1, length list) list - fix c = realign c arr - --- Realign a Ranged from a tabstop of 8 to 1 -realign :: Ranged a => a -> Array Int String -> a -realign range ls = - let startColumn = realignColumn lineNo colNo range - endColumn = realignColumn endLineNo endColNo range - startPosition = (start range) { posColumn = startColumn } - endPosition = (end range) { posColumn = endColumn } in - setRange (startPosition, endPosition) range - where - realignColumn lineNo colNo c = - if lineNo c > 0 && lineNo c <= fromIntegral (length ls) - then real (ls ! fromIntegral (lineNo c)) 0 0 (colNo c) - else colNo c - real _ r v target | target <= v = r - -- hit this case at the end of line, and if we don't hit the target - -- return real + (target - v) - real [] r v target = r + (target - v) - real ('\t':rest) r v target = real rest (r+1) (v + 8 - (v `mod` 8)) target - real (_:rest) r v target = real rest (r+1) (v+1) target - lineNo = posLine . start - endLineNo = posLine . end - colNo = posColumn . start - endColNo = posColumn . end + fix c = removeTabStops c arr diff --git a/src/ShellCheck/Formatter/TTY.hs b/src/ShellCheck/Formatter/TTY.hs index 254287f..4e9c272 100644 --- a/src/ShellCheck/Formatter/TTY.hs +++ b/src/ShellCheck/Formatter/TTY.hs @@ -163,25 +163,13 @@ showFixedString color comments lineNum fileLines = [] -> return () fixes -> do -- Folding automatically removes overlap - let mergedFix = realignFix $ fold fixes - -- We show the complete, associated fixes, whether or not it includes this and/or unrelated lines. + let mergedFix = fold fixes + -- We show the complete, associated fixes, whether or not it includes this + -- and/or other unrelated lines. let (excerptFix, excerpt) = sliceFile mergedFix fileLines -- in the spirit of error prone putStrLn $ color "message" "Did you mean: " - putStrLn $ unlines $ fixedString excerptFix excerpt - where - -- FIXME: This should be handled by Fixer - realignFix f = f { fixReplacements = map fix (fixReplacements f) } - fix r = realign r fileLines - -fixedString :: Fix -> Array Int String -> [String] -fixedString fix fileLines = - case (fixReplacements fix) of - [] -> [] - reps -> - -- applyReplacement returns the full update file, we really only care about the changed lines - -- so we calculate overlapping lines using replacements - applyFix fix fileLines + putStrLn $ unlines $ applyFix excerptFix excerpt cuteIndent :: PositionedComment -> String cuteIndent comment = From fcdd6055df1600d18466aa900dcdfddef9fc299e Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 9 Jan 2019 18:35:36 -0800 Subject: [PATCH 057/763] Add new replacement format to the JSON --- src/ShellCheck/Formatter/JSON.hs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Formatter/JSON.hs b/src/ShellCheck/Formatter/JSON.hs index 9aec751..02a549d 100644 --- a/src/ShellCheck/Formatter/JSON.hs +++ b/src/ShellCheck/Formatter/JSON.hs @@ -45,11 +45,16 @@ instance ToJSON Replacement where end = repEndPos replacement str = repString replacement in object [ + "precedence" .= repPrecedence replacement, + "insertionPoint" .= + case repInsertionPoint replacement of + InsertBefore -> "beforeStart" :: String + InsertAfter -> "afterEnd", "line" .= posLine start, - "endLine" .= posLine end, "column" .= posColumn start, + "endLine" .= posLine end, "endColumn" .= posColumn end, - "replaceWith" .= str + "replacement" .= str ] instance ToJSON PositionedComment where From 3760e7945f37944c1b1c4f2b73e1c49ac53e4bb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristian=20Adri=C3=A1n=20Ontivero?= Date: Thu, 10 Jan 2019 11:03:33 +0100 Subject: [PATCH 058/763] Check readonly flags in dash/POSIX sh (fixes #1448) --- src/ShellCheck/Checks/ShellSupport.hs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 674256e..7f30224 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -143,7 +143,8 @@ prop_checkBashisms60 = verifyNot checkBashisms "#!/bin/sh\njobs -p" prop_checkBashisms61 = verifyNot checkBashisms "#!/bin/sh\njobs -lp" prop_checkBashisms62 = verify checkBashisms "#!/bin/sh\nexport -f foo" prop_checkBashisms63 = verifyNot checkBashisms "#!/bin/sh\nexport -p" - +prop_checkBashisms64 = verify checkBashisms "#!/bin/sh\nreadonly -a" +prop_checkBashisms65 = verifyNot checkBashisms "#!/bin/sh\nreadonly -p" checkBashisms = ForShell [Sh, Dash] $ \t -> do params <- ask kludge params t @@ -292,6 +293,7 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do ("jobs", ["l", "p"]), ("printf", []), ("read", if isDash then ["r", "p"] else ["r"]), + ("readonly", ["p"]), ("ulimit", if isDash then ["H", "S", "t", "f", "d", "s", "c", "m", "l", "p", "n"] else ["f"]) ] bashism t@(T_SourceCommand id src _) = From b34f4c1f4baa62265a0e8392fd9e81cdb58f8476 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristian=20Adri=C3=A1n=20Ontivero?= Date: Sun, 13 Jan 2019 16:18:39 +0100 Subject: [PATCH 059/763] Silence SC2103 when using 'set -e' (fixes #667) --- src/ShellCheck/Analytics.hs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index c82fe31..26f198c 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2133,6 +2133,8 @@ prop_checkCdAndBack2 = verifyNot checkCdAndBack "for f in *; do cd $f || continu prop_checkCdAndBack3 = verifyNot checkCdAndBack "while [[ $PWD != / ]]; do cd ..; done" prop_checkCdAndBack4 = verify checkCdAndBack "cd $tmp; foo; cd -" prop_checkCdAndBack5 = verifyNot checkCdAndBack "cd ..; foo; cd .." +prop_checkCdAndBack6 = verify checkCdAndBack "for dir in */; do cd \"$dir\"; some_cmd; cd ..; done" +prop_checkCdAndBack7 = verifyNot checkCdAndBack "set -e; for dir in */; do cd \"$dir\"; some_cmd; cd ..; done" checkCdAndBack params = doLists where shell = shellType params @@ -2163,14 +2165,13 @@ checkCdAndBack params = doLists else findCdPair (b:rest) _ -> Nothing - doList list = - let cds = filter ((== Just "cd") . getCmd) list in - potentially $ do - cd <- findCdPair cds - return $ info cd 2103 message - - message = "Use a ( subshell ) to avoid having to cd back." + if hasSetE params + then return () + else let cds = filter ((== Just "cd") . getCmd) list + in potentially $ do + cd <- findCdPair cds + return $ info cd 2103 "Use a ( subshell ) to avoid having to cd back." prop_checkLoopKeywordScope1 = verify checkLoopKeywordScope "continue 2" prop_checkLoopKeywordScope2 = verify checkLoopKeywordScope "for f; do ( break; ); done" From 1835ebd3a0e5ff1460ad7fc63c228905ac5ff1f7 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 13 Jan 2019 16:06:36 -0800 Subject: [PATCH 060/763] SC2245: Warn that Ksh [ -f * ] only applies to first (Fixes #1452) --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 11e1994..71f1a88 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## Since previous release ### Added - Preliminary support for fix suggestions +- SC2245: Warn that Ksh ignores all but the first glob result in `[` - SC2243/SC2244: Suggest using explicit -n for `[ $foo ]` ## v0.6.0 - 2018-12-02 diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index c82fe31..2631a58 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2466,8 +2466,10 @@ prop_checkTestArgumentSplitting13 = verify checkTestArgumentSplitting "[ \"$@\" prop_checkTestArgumentSplitting14 = verify checkTestArgumentSplitting "[[ \"$@\" == \"\" ]]" prop_checkTestArgumentSplitting15 = verifyNot checkTestArgumentSplitting "[[ \"$*\" == \"\" ]]" prop_checkTestArgumentSplitting16 = verifyNot checkTestArgumentSplitting "[[ -v foo[123] ]]" +prop_checkTestArgumentSplitting17 = verifyNot checkTestArgumentSplitting "#!/bin/ksh\n[ -e foo* ]" +prop_checkTestArgumentSplitting18 = verify checkTestArgumentSplitting "#!/bin/ksh\n[ -d foo* ]" checkTestArgumentSplitting :: Parameters -> Token -> Writer [TokenComment] () -checkTestArgumentSplitting _ t = +checkTestArgumentSplitting params t = case t of (TC_Unary _ typ op token) | isGlob token -> if op == "-v" @@ -2476,8 +2478,16 @@ checkTestArgumentSplitting _ t = err (getId token) 2208 $ "Use [[ ]] or quote arguments to -v to avoid glob expansion." else - err (getId token) 2144 $ - op ++ " doesn't work with globs. Use a for loop." + if (typ == SingleBracket && shellType params == Ksh) + then + -- Ksh appears to stop processing after unrecognized tokens, so operators + -- will effectively work with globs, but only the first match. + when (op `elem` ['-':c:[] | c <- "bcdfgkprsuwxLhNOGRS" ]) $ + warn (getId token) 2245 $ + op ++ " only applies to the first expansion of this glob. Use a loop to check any/all." + else + err (getId token) 2144 $ + op ++ " doesn't work with globs. Use a for loop." (TC_Nullary _ typ token) -> do checkBraces typ token From e0a4241baa6e6ca72c9eedb8150cc8715240b05b Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 13 Jan 2019 17:32:25 -0800 Subject: [PATCH 061/763] Warn if a shebang's interpreter ends in / (fixes #373) --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 11 +++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 71f1a88..69cb66c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## Since previous release ### Added - Preliminary support for fix suggestions +- SC2246: Warn if a shebang's interpreter ends with / - SC2245: Warn that Ksh ignores all but the first glob result in `[` - SC2243/SC2244: Suggest using explicit -n for `[ $foo ]` diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 2631a58..6230f5b 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -474,6 +474,8 @@ prop_checkShebang7 = verifyNotTree checkShebang "#!/usr/bin/env ash\n# shellchec prop_checkShebang8 = verifyTree checkShebang "#!bin/sh\ntrue" prop_checkShebang9 = verifyNotTree checkShebang "# shellcheck shell=sh\ntrue" prop_checkShebang10= verifyNotTree checkShebang "#!foo\n# shellcheck shell=sh ignore=SC2239\ntrue" +prop_checkShebang11= verifyTree checkShebang "#!/bin/sh/\ntrue" +prop_checkShebang12= verifyTree checkShebang "#!/bin/sh/ -xe\ntrue" checkShebang params (T_Annotation _ list t) = if any isOverride list then [] else checkShebang params t where @@ -485,8 +487,13 @@ checkShebang params (T_Script id sb _) = execWriter $ do err id 2148 "Tips depend on target shell and yours is unknown. Add a shebang." when (executableFromShebang sb == "ash") $ warn id 2187 "Ash scripts will be checked as Dash. Add '# shellcheck shell=dash' to silence." - unless (null sb || "/" `isPrefixOf` sb) $ - err id 2239 "Ensure the shebang uses an absolute path to the interpreter." + unless (null sb) $ do + unless ("/" `isPrefixOf` sb) $ + err id 2239 "Ensure the shebang uses an absolute path to the interpreter." + case words sb of + first:_ -> + when ("/" `isSuffixOf` first) $ + err id 2246 "This shebang specifies a directory. Ensure the interpreter is a file." prop_checkForInQuoted = verify checkForInQuoted "for f in \"$(ls)\"; do echo foo; done" From c3a56659f4728163a5731b66535260ede38cf701 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristian=20Adri=C3=A1n=20Ontivero?= Date: Mon, 14 Jan 2019 08:18:17 +0100 Subject: [PATCH 062/763] Check cd flags under dash & POSIX sh (fixes #1457) --- src/ShellCheck/Checks/ShellSupport.hs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 7f30224..64bb383 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -145,6 +145,8 @@ prop_checkBashisms62 = verify checkBashisms "#!/bin/sh\nexport -f foo" prop_checkBashisms63 = verifyNot checkBashisms "#!/bin/sh\nexport -p" prop_checkBashisms64 = verify checkBashisms "#!/bin/sh\nreadonly -a" prop_checkBashisms65 = verifyNot checkBashisms "#!/bin/sh\nreadonly -p" +prop_checkBashisms66 = verifyNot checkBashisms "#!/bin/sh\ncd -P ." +prop_checkBashisms67 = verify checkBashisms "#!/bin/sh\ncd -P -e ." checkBashisms = ForShell [Sh, Dash] $ \t -> do params <- ask kludge params t @@ -288,6 +290,7 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do "typeset" ] ++ if not isDash then ["local"] else [] allowedFlags = Map.fromList [ + ("cd", ["L", "P"]), ("exec", []), ("export", ["p"]), ("jobs", ["l", "p"]), From a4b9cec9f00ba31e8b48c379296d2bb1d4bb3638 Mon Sep 17 00:00:00 2001 From: Tito Sacchi Date: Mon, 14 Jan 2019 14:16:12 +0100 Subject: [PATCH 063/763] Fix #1369 (Use file extension to detect shell) The precedence order that is used to determine the shell is the following: 1. ShellCheck directive 2. Shebang 3. File extension A new field `asFallbackShell` has been added to the record type `AnalysisSpec`. --- src/ShellCheck/AnalyzerLib.hs | 10 ++++++---- src/ShellCheck/Checker.hs | 10 ++++++++++ src/ShellCheck/Interface.hs | 4 +++- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 37a96ad..5248d94 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -175,7 +175,7 @@ makeCommentWithFix severity id code str fix = makeParameters spec = let params = Parameters { rootNode = root, - shellType = fromMaybe (determineShell root) $ asShellType spec, + shellType = fromMaybe (determineShell (asFallbackShell spec) root) $ asShellType spec, hasSetE = containsSetE root, hasLastpipe = case shellType params of @@ -227,11 +227,13 @@ prop_determineShell4 = determineShellTest "#!/bin/ksh\n#shellcheck shell=sh\nfoo prop_determineShell5 = determineShellTest "#shellcheck shell=sh\nfoo" == Sh prop_determineShell6 = determineShellTest "#! /bin/sh" == Sh prop_determineShell7 = determineShellTest "#! /bin/ash" == Dash +prop_determineShell8 = determineShellTest' (Just Ksh) "#!/bin/sh" == Sh -determineShellTest = determineShell . fromJust . prRoot . pScript -determineShell t = fromMaybe Bash $ do +determineShellTest = determineShellTest' Nothing +determineShellTest' fallbackShell = determineShell fallbackShell . fromJust . prRoot . pScript +determineShell fallbackShell t = fromMaybe Bash $ do shellString <- foldl mplus Nothing $ getCandidates t - shellForExecutable shellString + shellForExecutable shellString `mplus` fallbackShell where forAnnotation t = case t of diff --git a/src/ShellCheck/Checker.hs b/src/ShellCheck/Checker.hs index 10074e3..67bd1c3 100644 --- a/src/ShellCheck/Checker.hs +++ b/src/ShellCheck/Checker.hs @@ -48,6 +48,15 @@ tokenToPosition startMap t = fromMaybe fail $ do where fail = error "Internal shellcheck error: id doesn't exist. Please report!" +shellFromFilename filename = foldl mplus Nothing candidates + where + shellExtensions = [(".ksh", Ksh) + ,(".sh", Sh) + ,(".bash", Bash) + ,(".dash", Dash)] + candidates = + map (\(ext,sh) -> if ext `isSuffixOf` filename then Just sh else Nothing) shellExtensions + checkScript :: Monad m => SystemInterface m -> CheckSpec -> m CheckResult checkScript sys spec = do results <- checkScript (csScript spec) @@ -69,6 +78,7 @@ checkScript sys spec = do as { asScript = root, asShellType = csShellTypeOverride spec, + asFallbackShell = shellFromFilename $ csFilename spec, asCheckSourced = csCheckSourced spec, asExecutionMode = Executed, asTokenPositions = tokenPositions diff --git a/src/ShellCheck/Interface.hs b/src/ShellCheck/Interface.hs index dd53f6f..285042f 100644 --- a/src/ShellCheck/Interface.hs +++ b/src/ShellCheck/Interface.hs @@ -25,7 +25,7 @@ module ShellCheck.Interface , CheckResult(crFilename, crComments) , ParseSpec(psFilename, psScript, psCheckSourced, psShellTypeOverride) , ParseResult(prComments, prTokenPositions, prRoot) - , AnalysisSpec(asScript, asShellType, asExecutionMode, asCheckSourced, asTokenPositions) + , AnalysisSpec(asScript, asShellType, asFallbackShell, asExecutionMode, asCheckSourced, asTokenPositions) , AnalysisResult(arComments) , FormatterOptions(foColorOption, foWikiLinkCount) , Shell(Ksh, Sh, Bash, Dash) @@ -138,6 +138,7 @@ newParseResult = ParseResult { data AnalysisSpec = AnalysisSpec { asScript :: Token, asShellType :: Maybe Shell, + asFallbackShell :: Maybe Shell, asExecutionMode :: ExecutionMode, asCheckSourced :: Bool, asTokenPositions :: Map.Map Id (Position, Position) @@ -146,6 +147,7 @@ data AnalysisSpec = AnalysisSpec { newAnalysisSpec token = AnalysisSpec { asScript = token, asShellType = Nothing, + asFallbackShell = Nothing, asExecutionMode = Executed, asCheckSourced = False, asTokenPositions = Map.empty From 1e6a30905a986962a6d78ec55609168a724d253a Mon Sep 17 00:00:00 2001 From: Tito Sacchi Date: Mon, 14 Jan 2019 14:25:01 +0100 Subject: [PATCH 064/763] Make ShellCheck not emit warnings about the shebang if the shell type is determined from the extension --- src/ShellCheck/Analytics.hs | 2 +- src/ShellCheck/AnalyzerLib.hs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 6230f5b..8f66aaf 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -484,7 +484,7 @@ checkShebang params (T_Annotation _ list t) = checkShebang params (T_Script id sb _) = execWriter $ do unless (shellTypeSpecified params) $ do when (sb == "") $ - err id 2148 "Tips depend on target shell and yours is unknown. Add a shebang." + err id 2148 "Tips depend on target shell and yours is unknown. Add a shebang or an extension to the filename." when (executableFromShebang sb == "ash") $ warn id 2187 "Ash scripts will be checked as Dash. Add '# shellcheck shell=dash' to silence." unless (null sb) $ do diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 5248d94..b2b4edd 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -184,7 +184,7 @@ makeParameters spec = Sh -> False Ksh -> True, - shellTypeSpecified = isJust $ asShellType spec, + shellTypeSpecified = isJust (asShellType spec) || isJust (asFallbackShell spec), parentMap = getParentTree root, variableFlow = getVariableFlow params root, tokenPositions = asTokenPositions spec From 3107a1bae0a7c40339801ce0dfb3e1e4d79ffc27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristian=20Adri=C3=A1n=20Ontivero?= Date: Sun, 13 Jan 2019 06:13:09 +0100 Subject: [PATCH 065/763] Check umask flags under dash & POSIX sh (fixes #1459) --- src/ShellCheck/Checks/ShellSupport.hs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 64bb383..644cee2 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -147,6 +147,8 @@ prop_checkBashisms64 = verify checkBashisms "#!/bin/sh\nreadonly -a" prop_checkBashisms65 = verifyNot checkBashisms "#!/bin/sh\nreadonly -p" prop_checkBashisms66 = verifyNot checkBashisms "#!/bin/sh\ncd -P ." prop_checkBashisms67 = verify checkBashisms "#!/bin/sh\ncd -P -e ." +prop_checkBashisms68 = verify checkBashisms "#!/bin/sh\numask -p" +prop_checkBashisms69 = verifyNot checkBashisms "#!/bin/sh\numask -S" checkBashisms = ForShell [Sh, Dash] $ \t -> do params <- ask kludge params t @@ -297,7 +299,8 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do ("printf", []), ("read", if isDash then ["r", "p"] else ["r"]), ("readonly", ["p"]), - ("ulimit", if isDash then ["H", "S", "t", "f", "d", "s", "c", "m", "l", "p", "n"] else ["f"]) + ("ulimit", if isDash then ["H", "S", "t", "f", "d", "s", "c", "m", "l", "p", "n"] else ["f"]), + ("umask", ["S"]) ] bashism t@(T_SourceCommand id src _) = let name = fromMaybe "" $ getCommandName src From c6c615217b64a802f4a6a8f9be9832eed7b6139e Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 15 Jan 2019 19:19:45 -0800 Subject: [PATCH 066/763] Allow specifying that flags should not be checked for support. This was motivated by the fact that `-a` was missing from Dash's long list. --- src/ShellCheck/Checks/ShellSupport.hs | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 644cee2..2ffe30d 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -257,7 +257,8 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do when (name `elem` unsupportedCommands) $ warnMsg id $ "'" ++ name ++ "' is" potentially $ do - allowed <- Map.lookup name allowedFlags + allowed' <- Map.lookup name allowedFlags + allowed <- allowed' (word, flag) <- listToMaybe $ filter (\x -> (not . null . snd $ x) && snd x `notElem` allowed) flags return . warnMsg (getId word) $ name ++ " -" ++ flag ++ " is" @@ -292,20 +293,19 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do "typeset" ] ++ if not isDash then ["local"] else [] allowedFlags = Map.fromList [ - ("cd", ["L", "P"]), - ("exec", []), - ("export", ["p"]), - ("jobs", ["l", "p"]), - ("printf", []), - ("read", if isDash then ["r", "p"] else ["r"]), - ("readonly", ["p"]), - ("ulimit", if isDash then ["H", "S", "t", "f", "d", "s", "c", "m", "l", "p", "n"] else ["f"]), - ("umask", ["S"]) + ("cd", Just ["L", "P"]), + ("exec", Just []), + ("export", Just ["p"]), + ("jobs", Just ["l", "p"]), + ("printf", Just []), + ("read", Just $ if isDash then ["r", "p"] else ["r"]), + ("readonly", Just ["p"]), + ("ulimit", if isDash then Nothing else Just ["f"]), + ("umask", Just ["S"]) ] bashism t@(T_SourceCommand id src _) = let name = fromMaybe "" $ getCommandName src - in do - when (name == "source") $ warnMsg id "'source' in place of '.' is" + in when (name == "source") $ warnMsg id "'source' in place of '.' is" bashism _ = return () varChars="_0-9a-zA-Z" From 8e31e86cc4de92373f5a071ea9b4c523cc36a575 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristian=20Adri=C3=A1n=20Ontivero?= Date: Wed, 16 Jan 2019 08:44:41 +0100 Subject: [PATCH 067/763] Check trap flags under dash & POSIX sh (fixes #1461) --- src/ShellCheck/Checks/ShellSupport.hs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 2ffe30d..1ea1681 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -149,6 +149,7 @@ prop_checkBashisms66 = verifyNot checkBashisms "#!/bin/sh\ncd -P ." prop_checkBashisms67 = verify checkBashisms "#!/bin/sh\ncd -P -e ." prop_checkBashisms68 = verify checkBashisms "#!/bin/sh\numask -p" prop_checkBashisms69 = verifyNot checkBashisms "#!/bin/sh\numask -S" +prop_checkBashisms70 = verify checkBashisms "#!/bin/sh\ntrap -l" checkBashisms = ForShell [Sh, Dash] $ \t -> do params <- ask kludge params t @@ -300,6 +301,7 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do ("printf", Just []), ("read", Just $ if isDash then ["r", "p"] else ["r"]), ("readonly", Just ["p"]), + ("trap", Just []), ("ulimit", if isDash then Nothing else Just ["f"]), ("umask", Just ["S"]) ] From 9f45dc4c8b50f5086c67f67358c83a442afd1657 Mon Sep 17 00:00:00 2001 From: Tito Sacchi Date: Fri, 18 Jan 2019 09:21:07 +0100 Subject: [PATCH 068/763] Not determine the shell from `.sh` extension See discussion on issue #1369 for details. --- src/ShellCheck/Analytics.hs | 2 +- src/ShellCheck/Checker.hs | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 8f66aaf..78b5366 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -484,7 +484,7 @@ checkShebang params (T_Annotation _ list t) = checkShebang params (T_Script id sb _) = execWriter $ do unless (shellTypeSpecified params) $ do when (sb == "") $ - err id 2148 "Tips depend on target shell and yours is unknown. Add a shebang or an extension to the filename." + err id 2148 "Tips depend on target shell and yours is unknown. Add a shebang or a .bash, .ksh, .dash extension to the filename." when (executableFromShebang sb == "ash") $ warn id 2187 "Ash scripts will be checked as Dash. Add '# shellcheck shell=dash' to silence." unless (null sb) $ do diff --git a/src/ShellCheck/Checker.hs b/src/ShellCheck/Checker.hs index 67bd1c3..ac51ddd 100644 --- a/src/ShellCheck/Checker.hs +++ b/src/ShellCheck/Checker.hs @@ -51,9 +51,10 @@ tokenToPosition startMap t = fromMaybe fail $ do shellFromFilename filename = foldl mplus Nothing candidates where shellExtensions = [(".ksh", Ksh) - ,(".sh", Sh) ,(".bash", Bash) ,(".dash", Dash)] + -- The `.sh` is too generic to determine the shell: + -- We fallback to Bash in this case and emit SC2148 if there is no shebang candidates = map (\(ext,sh) -> if ext `isSuffixOf` filename then Just sh else Nothing) shellExtensions From 661be056f1cf8bbcb04faf73fbda09d06d6addbb Mon Sep 17 00:00:00 2001 From: Gandalf- Date: Sat, 19 Jan 2019 08:49:26 -0800 Subject: [PATCH 069/763] Issue 824 grep fixed strings and SC2063 Issue https://github.com/koalaman/shellcheck/issues/824 Fix up to original change to include '--fixed-strings' in the grep + regex special cases. --- src/ShellCheck/Checks/Commands.hs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 6c82a4f..66f597b 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -212,6 +212,7 @@ prop_checkGrepRe16= verifyNot checkGrepRe "grep --include 'Foo*' file" prop_checkGrepRe17= verifyNot checkGrepRe "grep --exclude 'Foo*' file" prop_checkGrepRe18= verifyNot checkGrepRe "grep --exclude-dir 'Foo*' file" prop_checkGrepRe19= verify checkGrepRe "grep -- 'Foo*' file" +prop_checkGrepRe20= verifyNot checkGrepRe "grep --fixed-strings 'Foo*' file" checkGrepRe = CommandCheck (Basename "grep") check where check cmd = f cmd (arguments cmd) @@ -244,7 +245,7 @@ checkGrepRe = CommandCheck (Basename "grep") check where "Note that unlike globs, " ++ [char] ++ "* here matches '" ++ [char, char, char] ++ "' but not '" ++ wordStartingWith char ++ "'." where flags = map snd $ getAllFlags cmd - grepGlobFlags = ["F", "include", "exclude", "exclude-dir"] + grepGlobFlags = ["fixed-strings", "F", "include", "exclude", "exclude-dir"] wordStartingWith c = head . filter ([c] `isPrefixOf`) $ candidates From f187382a0cdbae19b121e05ab8d141e420d507b7 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 22 May 2016 18:53:35 -0700 Subject: [PATCH 070/763] Add bats support This is motivated by the fact that the popularity of bats is increasing since the creation of bats-core/bats-core. The code is a cherry-pick of koalaman/shellcheck/bats branch. Fix koalaman/shellcheck#417. --- src/ShellCheck/AST.hs | 3 +++ src/ShellCheck/Analytics.hs | 7 ++++++- src/ShellCheck/AnalyzerLib.hs | 14 ++++++++++++++ src/ShellCheck/Data.hs | 1 + src/ShellCheck/Parser.hs | 13 +++++++++++++ 5 files changed, 37 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/AST.hs b/src/ShellCheck/AST.hs index 8a6d7b2..aedb148 100644 --- a/src/ShellCheck/AST.hs +++ b/src/ShellCheck/AST.hs @@ -139,6 +139,7 @@ data Token = | T_CoProcBody Id Token | T_Include Id Token | T_SourceCommand Id Token Token + | T_BatsTest Id Token Token deriving (Show) data Annotation = @@ -276,6 +277,7 @@ analyze f g i = delve (T_CoProcBody id t) = d1 t $ T_CoProcBody id delve (T_Include id script) = d1 script $ T_Include id delve (T_SourceCommand id includer t_include) = d2 includer t_include $ T_SourceCommand id + delve (T_BatsTest id name t) = d2 name t $ T_BatsTest id delve t = return t getId :: Token -> Id @@ -380,6 +382,7 @@ getId t = case t of T_UnparsedIndex id _ _ -> id TC_Empty id _ -> id TA_Variable id _ _ -> id + T_BatsTest id _ _ -> id blank :: Monad m => Token -> m () blank = const $ return () diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 6230f5b..e057449 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -232,7 +232,9 @@ hasFloatingPoint params = shellType params == Ksh isCondition [] = False isCondition [_] = False isCondition (child:parent:rest) = - getId child `elem` map getId (getConditionChildren parent) || isCondition (parent:rest) + case child of + T_BatsTest {} -> True -- count anything in a @test as conditional + _ -> getId child `elem` map getId (getConditionChildren parent) || isCondition (parent:rest) where getConditionChildren t = case t of @@ -1580,6 +1582,7 @@ prop_subshellAssignmentCheck16 = verifyNotTree subshellAssignmentCheck "(set -e) prop_subshellAssignmentCheck17 = verifyNotTree subshellAssignmentCheck "foo=${ { bar=$(baz); } 2>&1; }; echo $foo $bar" prop_subshellAssignmentCheck18 = verifyTree subshellAssignmentCheck "( exec {n}>&2; ); echo $n" prop_subshellAssignmentCheck19 = verifyNotTree subshellAssignmentCheck "#!/bin/bash\nshopt -s lastpipe; echo a | read -r b; echo \"$b\"" +prop_subshellAssignmentCheck20 = verifyTree subshellAssignmentCheck "@test 'foo' { a=1; }\n@test 'bar' { echo $a; }\n" subshellAssignmentCheck params t = let flow = variableFlow params check = findSubshelled flow [("oops",[])] Map.empty @@ -1666,6 +1669,7 @@ prop_checkSpacefulness33= verifyTree checkSpacefulness "for file; do echo $file; prop_checkSpacefulness34= verifyTree checkSpacefulness "declare foo$n=$1" prop_checkSpacefulness35= verifyNotTree checkSpacefulness "echo ${1+\"$1\"}" prop_checkSpacefulness36= verifyNotTree checkSpacefulness "arg=$#; echo $arg" +prop_checkSpacefulness37= verifyNotTree checkSpacefulness "@test 'status' {\n [ $status -eq 0 ]\n}" checkSpacefulness params t = doVariableFlowAnalysis readF writeF (Map.fromList defaults) (variableFlow params) @@ -1891,6 +1895,7 @@ prop_checkUnused37= verifyNotTree checkUnusedAssignments "fd=2; exec {fd}>&-" prop_checkUnused38= verifyTree checkUnusedAssignments "(( a=42 ))" prop_checkUnused39= verifyNotTree checkUnusedAssignments "declare -x -f foo" prop_checkUnused40= verifyNotTree checkUnusedAssignments "arr=(1 2); num=2; echo \"${arr[@]:num}\"" +prop_checkUnused41= verifyNotTree checkUnusedAssignments "@test 'foo' {\ntrue\n}\n" checkUnusedAssignments params t = execWriter (mapM_ warnFor unused) where flow = variableFlow params diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 37a96ad..e0c07f4 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -423,6 +423,7 @@ getVariableFlow params t = assignFirst T_ForIn {} = True assignFirst T_SelectIn {} = True + assignFirst (T_BatsTest {}) = True assignFirst _ = False setRead t = @@ -440,6 +441,7 @@ leadType params t = T_Backticked _ _ -> SubshellScope "`..` expansion" T_Backgrounded _ _ -> SubshellScope "backgrounding &" T_Subshell _ _ -> SubshellScope "(..) group" + T_BatsTest {} -> SubshellScope "@bats test" T_CoProcBody _ _ -> SubshellScope "coproc" T_Redirecting {} -> if fromMaybe False causesSubshell @@ -480,6 +482,12 @@ getModifiedVariables t = guard $ op `elem` ["=", "*=", "/=", "%=", "+=", "-=", "<<=", ">>=", "&=", "^=", "|="] return (t, t, name, DataString $ SourceFrom [rhs]) + T_BatsTest {} -> [ + (t, t, "lines", DataArray SourceExternal), + (t, t, "status", DataString SourceInteger), + (t, t, "output", DataString SourceExternal) + ] + -- Count [[ -v foo ]] as an "assignment". -- This is to prevent [ -v foo ] being unassigned or unused. TC_Unary id _ "-v" token -> maybeToList $ do @@ -699,6 +707,12 @@ getReferencedVariables parents t = then concatMap (getIfReference t) [lhs, rhs] else [] + T_BatsTest {} -> [ -- pretend @test references vars to avoid warnings + (t, t, "lines"), + (t, t, "status"), + (t, t, "output") + ] + t@(T_FdRedirect _ ('{':var) op) -> -- {foo}>&- references and closes foo [(t, t, takeWhile (/= '}') var) | isClosingFileOp op] x -> getReferencedVariableCommand x diff --git a/src/ShellCheck/Data.hs b/src/ShellCheck/Data.hs index 8ce0026..e4ef675 100644 --- a/src/ShellCheck/Data.hs +++ b/src/ShellCheck/Data.hs @@ -109,6 +109,7 @@ shellForExecutable name = case name of "sh" -> return Sh "bash" -> return Bash + "bats" -> return Bash "dash" -> return Dash "ash" -> return Dash -- There's also a warning for this. "ksh" -> return Ksh diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 172ef54..14e31e3 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -2331,6 +2331,17 @@ readBraceGroup = called "brace group" $ do id <- endSpan start return $ T_BraceGroup id list +prop_readBatsTest = isOk readBatsTest "@test 'can parse' {\n true\n}" +readBatsTest = called "bats @test" $ do + start <- startSpan + try $ string "@test" + spacing + name <- readNormalWord + spacing + test <- readBraceGroup + id <- endSpan start + return $ T_BatsTest id name test + prop_readWhileClause = isOk readWhileClause "while [[ -e foo ]]; do sleep 1; done" readWhileClause = called "while loop" $ do start <- startSpan @@ -2590,6 +2601,7 @@ readCompoundCommand = do readForClause, readSelectClause, readCaseClause, + readBatsTest, readFunctionDefinition ] spacing @@ -3037,6 +3049,7 @@ readScriptFile = do "ash", "dash", "bash", + "bats", "ksh" ] badShells = [ From a504ca6b5721030a6d113043a0f2a4b0eac287e5 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 20 Jan 2019 13:24:31 -0800 Subject: [PATCH 071/763] Add some unit tests for extension detection --- src/ShellCheck/Analytics.hs | 2 +- src/ShellCheck/Checker.hs | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 78b5366..6230f5b 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -484,7 +484,7 @@ checkShebang params (T_Annotation _ list t) = checkShebang params (T_Script id sb _) = execWriter $ do unless (shellTypeSpecified params) $ do when (sb == "") $ - err id 2148 "Tips depend on target shell and yours is unknown. Add a shebang or a .bash, .ksh, .dash extension to the filename." + err id 2148 "Tips depend on target shell and yours is unknown. Add a shebang." when (executableFromShebang sb == "ash") $ warn id 2187 "Ash scripts will be checked as Dash. Add '# shellcheck shell=dash' to silence." unless (null sb) $ do diff --git a/src/ShellCheck/Checker.hs b/src/ShellCheck/Checker.hs index ac51ddd..2c5eea0 100644 --- a/src/ShellCheck/Checker.hs +++ b/src/ShellCheck/Checker.hs @@ -244,5 +244,35 @@ prop_sourcePartOfOriginalScript = -- #1181: -x disabled posix warning for 'sourc prop_spinBug1413 = null $ check "fun() {\n# shellcheck disable=SC2188\n> /dev/null\n}\n" +prop_deducesTypeFromExtension = null result + where + result = checkWithSpec [] emptyCheckSpec { + csFilename = "file.ksh", + csScript = "(( 3.14 ))" + } + +prop_deducesTypeFromExtension2 = result == [2079] + where + result = checkWithSpec [] emptyCheckSpec { + csFilename = "file.bash", + csScript = "(( 3.14 ))" + } + +prop_shExtensionDoesntMatter = result == [2148] + where + result = checkWithSpec [] emptyCheckSpec { + csFilename = "file.sh", + csScript = "echo 'hello world'" + } + +prop_sourcedFileUsesOriginalShellExtension = result == [2079] + where + result = checkWithSpec [("file.ksh", "(( 3.14 ))")] emptyCheckSpec { + csFilename = "file.bash", + csScript = "source file.ksh", + csCheckSourced = True + } + + return [] runTests = $quickCheckAll From e1fe9be7afba61bc4fb65e6a8828f54c6e21e552 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 20 Jan 2019 14:02:42 -0800 Subject: [PATCH 072/763] Fix minor details in new Bats support --- CHANGELOG.md | 1 + src/ShellCheck/ASTLib.hs | 8 ++++++++ src/ShellCheck/Checker.hs | 1 + src/ShellCheck/Checks/Commands.hs | 2 +- 4 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 69cb66c..7198770 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## Since previous release ### Added - Preliminary support for fix suggestions +- Files containing Bats tests can now be checked - SC2246: Warn if a shebang's interpreter ends with / - SC2245: Warn that Ksh ignores all but the first glob result in `[` - SC2243/SC2244: Suggest using explicit -n for `[ $foo ]` diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 5f3b68c..66269bc 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -351,6 +351,14 @@ isOnlyRedirection t = isFunction t = case t of T_Function {} -> True; _ -> False +-- Bats tests are functions for the purpose of 'local' and such +isFunctionLike t = + case t of + T_Function {} -> True + T_BatsTest {} -> True + _ -> False + + isBraceExpansion t = case t of T_BraceExpansion {} -> True; _ -> False -- Get the lists of commands from tokens that contain them, such as diff --git a/src/ShellCheck/Checker.hs b/src/ShellCheck/Checker.hs index 2c5eea0..b92cad3 100644 --- a/src/ShellCheck/Checker.hs +++ b/src/ShellCheck/Checker.hs @@ -52,6 +52,7 @@ shellFromFilename filename = foldl mplus Nothing candidates where shellExtensions = [(".ksh", Ksh) ,(".bash", Bash) + ,(".bats", Bash) ,(".dash", Dash)] -- The `.sh` is too generic to determine the shell: -- We fallback to Bash in this case and emit SC2148 if there is no shebang diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 6c82a4f..a121bc7 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -770,7 +770,7 @@ prop_checkLocalScope2 = verifyNot checkLocalScope "f() { local foo=3; }" checkLocalScope = CommandCheck (Exactly "local") $ \t -> whenShell [Bash, Dash] $ do -- Ksh allows it, Sh doesn't support local path <- getPathM t - unless (any isFunction path) $ + unless (any isFunctionLike path) $ err (getId $ getCommandTokenOrThis t) 2168 "'local' is only valid in functions." prop_checkDeprecatedTempfile1 = verify checkDeprecatedTempfile "var=$(tempfile)" From 59c47f2266431e331d57cfe295b4b2ff50ea5bcc Mon Sep 17 00:00:00 2001 From: Gandalf- Date: Fri, 18 Jan 2019 20:15:52 -0800 Subject: [PATCH 073/763] Issue 837 flag to include only certain warnings Issue https://github.com/koalaman/shellcheck/issues/837 Add an --include option, which creates a whitelist of warnings to report on, the opposite of --exclude. --- shellcheck.1.md | 7 +++++++ shellcheck.hs | 14 ++++++++++++++ src/ShellCheck/Checker.hs | 32 ++++++++++++++++++++++++++++---- src/ShellCheck/Interface.hs | 4 +++- 4 files changed, 52 insertions(+), 5 deletions(-) diff --git a/shellcheck.1.md b/shellcheck.1.md index 6600613..98fceca 100644 --- a/shellcheck.1.md +++ b/shellcheck.1.md @@ -44,6 +44,13 @@ not warn at all, as `ksh` supports decimals in arithmetic contexts. is *auto*. **--color** without an argument is equivalent to **--color=always**. +**-i**\ *CODE1*[,*CODE2*...],\ **--include=***CODE1*[,*CODE2*...] + +: Explicitly include only the specified codes in the report. Subsequent **-i** + options are cumulative, but all the codes can be specified at once, + comma-separated as a single argument. Include options override any provided + exclude options. + **-e**\ *CODE1*[,*CODE2*...],\ **--exclude=***CODE1*[,*CODE2*...] : Explicitly exclude the specified codes from the report. Subsequent **-e** diff --git a/shellcheck.hs b/shellcheck.hs index 84dee0a..dc9f742 100644 --- a/shellcheck.hs +++ b/shellcheck.hs @@ -87,6 +87,8 @@ options = [ Option "C" ["color"] (OptArg (maybe (Flag "color" "always") (Flag "color")) "WHEN") "Use color (auto, always, never)", + Option "i" ["include"] + (ReqArg (Flag "include") "CODE1,CODE2..") "Consider only given types of warnings", Option "e" ["exclude"] (ReqArg (Flag "exclude") "CODE1,CODE2..") "Exclude types of warnings", Option "f" ["format"] @@ -270,6 +272,18 @@ parseOption flag options = } } + Flag "include" str -> do + new <- mapM parseNum $ filter (not . null) $ split ',' str + let old = csIncludedWarnings . checkSpec $ options + return options { + checkSpec = (checkSpec options) { + csIncludedWarnings = + if null new + then old + else Just new `mappend` old + } + } + Flag "version" _ -> do liftIO printVersion throwError NoProblems diff --git a/src/ShellCheck/Checker.hs b/src/ShellCheck/Checker.hs index b92cad3..1efb073 100644 --- a/src/ShellCheck/Checker.hs +++ b/src/ShellCheck/Checker.hs @@ -94,11 +94,13 @@ checkScript sys spec = do (parseMessages ++ map translator analysisMessages) shouldInclude pc = - let code = cCode (pcComment pc) + severity <= csMinSeverity spec && + case csIncludedWarnings spec of + Nothing -> code `notElem` csExcludedWarnings spec + Just includedWarnings -> code `elem` includedWarnings + where + code = cCode (pcComment pc) severity = cSeverity (pcComment pc) - in - code `notElem` csExcludedWarnings spec && - severity <= csMinSeverity spec sortMessages = sortBy (comparing order) order pc = @@ -137,6 +139,13 @@ checkRecursive includes src = csCheckSourced = True } +checkOptionIncludes includes src = + checkWithSpec [] emptyCheckSpec { + csScript = src, + csIncludedWarnings = includes, + csCheckSourced = True + } + prop_findsParseIssue = check "echo \"$12\"" == [1037] prop_commentDisablesParseIssue1 = @@ -274,6 +283,21 @@ prop_sourcedFileUsesOriginalShellExtension = result == [2079] csCheckSourced = True } +prop_optionIncludes1 = + -- expect 2086, but not included, so not reported + null $ checkOptionIncludes (Just [2080]) "#!/bin/sh\n var='a b'\n echo $var" + +prop_optionIncludes2 = + -- expect 2086, included, so its reported + [2086] == checkOptionIncludes (Just [2086]) "#!/bin/sh\n var='a b'\n echo $var" + +prop_optionIncludes3 = + -- expect 2086, no inclusions provided, so its reported + [2086] == checkOptionIncludes Nothing "#!/bin/sh\n var='a b'\n echo $var" + +prop_optionIncludes4 = + -- expect 2086 & 2154, only 2154 included, so only its reported + [2154] == checkOptionIncludes (Just [2154]) "#!/bin/sh\n var='a b'\n echo $var\n echo $bar" return [] runTests = $quickCheckAll diff --git a/src/ShellCheck/Interface.hs b/src/ShellCheck/Interface.hs index 285042f..b492b3f 100644 --- a/src/ShellCheck/Interface.hs +++ b/src/ShellCheck/Interface.hs @@ -21,7 +21,7 @@ module ShellCheck.Interface ( SystemInterface(..) - , CheckSpec(csFilename, csScript, csCheckSourced, csExcludedWarnings, csShellTypeOverride, csMinSeverity) + , CheckSpec(csFilename, csScript, csCheckSourced, csIncludedWarnings, csExcludedWarnings, csShellTypeOverride, csMinSeverity) , CheckResult(crFilename, crComments) , ParseSpec(psFilename, psScript, psCheckSourced, psShellTypeOverride) , ParseResult(prComments, prTokenPositions, prRoot) @@ -80,6 +80,7 @@ data CheckSpec = CheckSpec { csScript :: String, csCheckSourced :: Bool, csExcludedWarnings :: [Integer], + csIncludedWarnings :: Maybe [Integer], csShellTypeOverride :: Maybe Shell, csMinSeverity :: Severity } deriving (Show, Eq) @@ -101,6 +102,7 @@ emptyCheckSpec = CheckSpec { csScript = "", csCheckSourced = False, csExcludedWarnings = [], + csIncludedWarnings = Nothing, csShellTypeOverride = Nothing, csMinSeverity = StyleC } From 63a259e5bed7c0f7f5287633e60f41605baa3e14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristian=20Adri=C3=A1n=20Ontivero?= Date: Mon, 21 Jan 2019 19:49:14 +0100 Subject: [PATCH 074/763] Check type flags under dash and POSIX sh (fixes #1471) There are no flags for the type builtin defined under POSIX sh, nor does dash define any. Bash, however, allows [-aftpP]. We check this now under POSIX and dash. --- src/ShellCheck/Checks/ShellSupport.hs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 1ea1681..8f55b3c 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -150,6 +150,8 @@ prop_checkBashisms67 = verify checkBashisms "#!/bin/sh\ncd -P -e ." prop_checkBashisms68 = verify checkBashisms "#!/bin/sh\numask -p" prop_checkBashisms69 = verifyNot checkBashisms "#!/bin/sh\numask -S" prop_checkBashisms70 = verify checkBashisms "#!/bin/sh\ntrap -l" +prop_checkBashisms71 = verify checkBashisms "#!/bin/sh\ntype -a ls" +prop_checkBashisms72 = verifyNot checkBashisms "#!/bin/sh\ntype ls" checkBashisms = ForShell [Sh, Dash] $ \t -> do params <- ask kludge params t @@ -302,6 +304,7 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do ("read", Just $ if isDash then ["r", "p"] else ["r"]), ("readonly", Just ["p"]), ("trap", Just []), + ("type", Just []), ("ulimit", if isDash then Nothing else Just ["f"]), ("umask", Just ["S"]) ] From 489c3a4ddf221b60c9e2f0476e38f96f08d88174 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 20 Jan 2019 15:06:33 -0800 Subject: [PATCH 075/763] Fix SC2164 always saying 'cd' even when using 'pushd' --- src/ShellCheck/Analytics.hs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 3265ab6..4e7b9c6 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2605,15 +2605,17 @@ checkUncheckedCdPushdPopd params root = [] else execWriter $ doAnalysis checkElement root where - checkElement t@T_SimpleCommand {} = - when(name t `elem` ["cd", "pushd", "popd"] + checkElement t@T_SimpleCommand {} = do + let name = getName t + when(name `elem` ["cd", "pushd", "popd"] && not (isSafeDir t) - && not (name t `elem` ["pushd", "popd"] && ("n" `elem` map snd (getAllFlags t))) + && not (name `elem` ["pushd", "popd"] && ("n" `elem` map snd (getAllFlags t))) && not (isCondition $ getPath (parentMap params) t)) $ - warnWithFix (getId t) 2164 "Use 'cd ... || exit' or 'cd ... || return' in case cd fails." + warnWithFix (getId t) 2164 + ("Use '" ++ name ++ " ... || exit' or '" ++ name ++ " ... || return' in case " ++ name ++ " fails.") (fixWith [replaceEnd (getId t) params 0 " || exit"]) checkElement _ = return () - name t = fromMaybe "" $ getCommandName t + getName t = fromMaybe "" $ getCommandName t isSafeDir t = case oversimplify t of [_, ".."] -> True; _ -> False From d4d219affde3894c2fc9a7c06cc366fe11f5603b Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 20 Jan 2019 21:38:10 -0800 Subject: [PATCH 076/763] Don't warn that `cd ../..` and similar can fail in SC2164 --- src/ShellCheck/Analytics.hs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 4e7b9c6..85b8bc6 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2599,6 +2599,10 @@ prop_checkUncheckedPopd6 = verifyTree checkUncheckedCdPushdPopd "popd" prop_checkUncheckedPopd7 = verifyNotTree checkUncheckedCdPushdPopd "#!/bin/bash -e\npopd\nrm bar" prop_checkUncheckedPopd8 = verifyNotTree checkUncheckedCdPushdPopd "set -o errexit; popd; rm bar" prop_checkUncheckedPopd9 = verifyNotTree checkUncheckedCdPushdPopd "popd -n foo" +prop_checkUncheckedPopd10 = verifyNotTree checkUncheckedCdPushdPopd "cd ../.." +prop_checkUncheckedPopd11 = verifyNotTree checkUncheckedCdPushdPopd "cd ../.././.." +prop_checkUncheckedPopd12 = verifyNotTree checkUncheckedCdPushdPopd "cd /" +prop_checkUncheckedPopd13 = verifyTree checkUncheckedCdPushdPopd "cd ../../.../.." checkUncheckedCdPushdPopd params root = if hasSetE params then @@ -2617,8 +2621,9 @@ checkUncheckedCdPushdPopd params root = checkElement _ = return () getName t = fromMaybe "" $ getCommandName t isSafeDir t = case oversimplify t of - [_, ".."] -> True; + [_, str] -> str `matches` regex _ -> False + regex = mkRegex "^/*((\\.|\\.\\.)/+)*(\\.|\\.\\.)?$" prop_checkLoopVariableReassignment1 = verify checkLoopVariableReassignment "for i in *; do for i in *.bar; do true; done; done" prop_checkLoopVariableReassignment2 = verify checkLoopVariableReassignment "for i in *; do for((i=0; i<3; i++)); do true; done; done" From 6dcf4b8e6457e2ea3e3de09e9e8bd37ca3408a3c Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 21 Jan 2019 16:54:28 -0800 Subject: [PATCH 077/763] Mention extension in changelog and man page --- CHANGELOG.md | 4 ++++ shellcheck.1.md | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7198770..5c6f54c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ - SC2245: Warn that Ksh ignores all but the first glob result in `[` - SC2243/SC2244: Suggest using explicit -n for `[ $foo ]` +### Changed +- If a directive or shebang is not specified, a `.bash/.bats/.dash/.ksh` + extension will be used to infer the shell type when present. + ## v0.6.0 - 2018-12-02 ### Added - Command line option --severity/-S for filtering by minimum severity diff --git a/shellcheck.1.md b/shellcheck.1.md index 6600613..01d8122 100644 --- a/shellcheck.1.md +++ b/shellcheck.1.md @@ -64,8 +64,8 @@ not warn at all, as `ksh` supports decimals in arithmetic contexts. **-s**\ *shell*,\ **--shell=***shell* : Specify Bourne shell dialect. Valid values are *sh*, *bash*, *dash* and *ksh*. - The default is to use the file's shebang, or *bash* if the target shell - can't be determined. + The default is to deduce the shell from the file's `shell` directive, + shebang, or `.bash/.bats/.dash/.ksh` extension, in that order. **-V**,\ **--version** From a89403f09b34e2742786a046f25217700839641e Mon Sep 17 00:00:00 2001 From: Gandalf- Date: Fri, 18 Jan 2019 19:21:17 -0800 Subject: [PATCH 078/763] Issue 1393 quiet flag Issue https://github.com/koalaman/shellcheck/issues/1393 Provide '-q' and '--quiet' flags that suppress all normal output, but keep the return status, similar to 'grep -q'. --- ShellCheck.cabal | 1 + shellcheck.1.md | 6 +++++ shellcheck.hs | 4 +++- src/ShellCheck/Formatter/Quiet.hs | 37 +++++++++++++++++++++++++++++++ 4 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 src/ShellCheck/Formatter/Quiet.hs diff --git a/ShellCheck.cabal b/ShellCheck.cabal index 3345e32..ff612ed 100644 --- a/ShellCheck.cabal +++ b/ShellCheck.cabal @@ -80,6 +80,7 @@ library ShellCheck.Formatter.GCC ShellCheck.Formatter.JSON ShellCheck.Formatter.TTY + ShellCheck.Formatter.Quiet ShellCheck.Interface ShellCheck.Parser ShellCheck.Regex diff --git a/shellcheck.1.md b/shellcheck.1.md index 6600613..10f9797 100644 --- a/shellcheck.1.md +++ b/shellcheck.1.md @@ -137,6 +137,12 @@ not warn at all, as `ksh` supports decimals in arithmetic contexts. ... ] +*quiet* + +: Suppress all normal output. Exit with zero if no issues are found, + otherwise exit with one. Stops processing after the first issue. + + # DIRECTIVES ShellCheck directives can be specified as comments in the shell script before a command or block: diff --git a/shellcheck.hs b/shellcheck.hs index 84dee0a..e3487c0 100644 --- a/shellcheck.hs +++ b/shellcheck.hs @@ -27,6 +27,7 @@ import ShellCheck.Formatter.Format import qualified ShellCheck.Formatter.GCC import qualified ShellCheck.Formatter.JSON import qualified ShellCheck.Formatter.TTY +import qualified ShellCheck.Formatter.Quiet import Control.Exception import Control.Monad @@ -125,7 +126,8 @@ formats options = Map.fromList [ ("checkstyle", ShellCheck.Formatter.CheckStyle.format), ("gcc", ShellCheck.Formatter.GCC.format), ("json", ShellCheck.Formatter.JSON.format), - ("tty", ShellCheck.Formatter.TTY.format options) + ("tty", ShellCheck.Formatter.TTY.format options), + ("quiet", ShellCheck.Formatter.Quiet.format options) ] formatList = intercalate ", " names diff --git a/src/ShellCheck/Formatter/Quiet.hs b/src/ShellCheck/Formatter/Quiet.hs new file mode 100644 index 0000000..9ad8b97 --- /dev/null +++ b/src/ShellCheck/Formatter/Quiet.hs @@ -0,0 +1,37 @@ +{- + Copyright 2019 Austin Voecks + + This file is part of ShellCheck. + https://www.shellcheck.net + + ShellCheck is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + ShellCheck is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +-} +module ShellCheck.Formatter.Quiet (format) where + +import ShellCheck.Interface +import ShellCheck.Formatter.Format + +import Control.Monad +import Data.IORef +import System.Exit + +format :: FormatterOptions -> IO Formatter +format options = do + topErrorRef <- newIORef [] + return Formatter { + header = return (), + footer = return (), + onFailure = \ _ _ -> exitFailure, + onResult = \ result _ -> unless (null $ crComments result) exitFailure + } From 2737496b3a5579cf07dd72998ff46433e9522ddf Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 22 Jan 2019 19:47:40 -0800 Subject: [PATCH 079/763] Fix grammatical error in comments --- src/ShellCheck/Checker.hs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/Checker.hs b/src/ShellCheck/Checker.hs index 1efb073..f9c1ede 100644 --- a/src/ShellCheck/Checker.hs +++ b/src/ShellCheck/Checker.hs @@ -284,19 +284,19 @@ prop_sourcedFileUsesOriginalShellExtension = result == [2079] } prop_optionIncludes1 = - -- expect 2086, but not included, so not reported + -- expect 2086, but not included, so nothing reported null $ checkOptionIncludes (Just [2080]) "#!/bin/sh\n var='a b'\n echo $var" prop_optionIncludes2 = - -- expect 2086, included, so its reported + -- expect 2086, included, so it is reported [2086] == checkOptionIncludes (Just [2086]) "#!/bin/sh\n var='a b'\n echo $var" prop_optionIncludes3 = - -- expect 2086, no inclusions provided, so its reported + -- expect 2086, no inclusions provided, so it is reported [2086] == checkOptionIncludes Nothing "#!/bin/sh\n var='a b'\n echo $var" prop_optionIncludes4 = - -- expect 2086 & 2154, only 2154 included, so only its reported + -- expect 2086 & 2154, only 2154 included, so only that's reported [2154] == checkOptionIncludes (Just [2154]) "#!/bin/sh\n var='a b'\n echo $var\n echo $bar" return [] From 31c5601c5efcc637bdcbac43d0479b985108d985 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristian=20Adri=C3=A1n=20Ontivero?= Date: Wed, 23 Jan 2019 06:35:08 +0100 Subject: [PATCH 080/763] Check unset flags under dash and POSIX sh The only acceptable flags for the unset builtin under POSIX sh and dash are [-fv]. Bash though, accepts [-n] too. This commits makes shellcheck warn about this. --- src/ShellCheck/Checks/ShellSupport.hs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 8f55b3c..96a63b8 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -152,6 +152,8 @@ prop_checkBashisms69 = verifyNot checkBashisms "#!/bin/sh\numask -S" prop_checkBashisms70 = verify checkBashisms "#!/bin/sh\ntrap -l" prop_checkBashisms71 = verify checkBashisms "#!/bin/sh\ntype -a ls" prop_checkBashisms72 = verifyNot checkBashisms "#!/bin/sh\ntype ls" +prop_checkBashisms73 = verify checkBashisms "#!/bin/sh\nunset -n namevar" +prop_checkBashisms74 = verifyNot checkBashisms "#!/bin/sh\nunset -f namevar" checkBashisms = ForShell [Sh, Dash] $ \t -> do params <- ask kludge params t @@ -306,7 +308,8 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do ("trap", Just []), ("type", Just []), ("ulimit", if isDash then Nothing else Just ["f"]), - ("umask", Just ["S"]) + ("umask", Just ["S"]), + ("unset", Just ["f", "v"]) ] bashism t@(T_SourceCommand id src _) = let name = fromMaybe "" $ getCommandName src From 112a7d8b9b819c38884527ada71539f3a9b745d7 Mon Sep 17 00:00:00 2001 From: Gandalf- Date: Fri, 11 Jan 2019 10:17:44 -0800 Subject: [PATCH 081/763] Issue 1330 unsupported echo flags Issue https://github.com/koalaman/shellcheck/issues/1330 Addresses false positives when quoted arguments to echo begin with what looks like a flag. Now, warn only when the first argument is a recognized echo flag when flags are unsupported. --- src/ShellCheck/Checks/ShellSupport.hs | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 8f55b3c..40397af 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -152,6 +152,10 @@ prop_checkBashisms69 = verifyNot checkBashisms "#!/bin/sh\numask -S" prop_checkBashisms70 = verify checkBashisms "#!/bin/sh\ntrap -l" prop_checkBashisms71 = verify checkBashisms "#!/bin/sh\ntype -a ls" prop_checkBashisms72 = verifyNot checkBashisms "#!/bin/sh\ntype ls" +prop_checkBashisms73 = verifyNot checkBashisms "#!/bin/sh\necho \"-n foo\"" +prop_checkBashisms74 = verifyNot checkBashisms "#!/bin/sh\necho \"-ne foo\"" +prop_checkBashisms75 = verifyNot checkBashisms "#!/bin/sh\necho -Q foo" +prop_checkBashisms76 = verify checkBashisms "#!/bin/sh\necho -ne foo" checkBashisms = ForShell [Sh, Dash] $ \t -> do params <- ask kludge params t @@ -238,15 +242,17 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do warnMsg id "` Date: Sun, 27 Jan 2019 08:22:37 +0100 Subject: [PATCH 082/763] Check hash flags under dash and POSIX sh Flags for the hash builtin other than [-r] are undefined under POSIX sh. Dash also accepts [-v], while bash adds [-l] [-p filename] [-dt] aside from [-r]. --- src/ShellCheck/Checks/ShellSupport.hs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 5a164d8..54af521 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -158,6 +158,9 @@ prop_checkBashisms75 = verifyNot checkBashisms "#!/bin/sh\necho \"-n foo\"" prop_checkBashisms76 = verifyNot checkBashisms "#!/bin/sh\necho \"-ne foo\"" prop_checkBashisms77 = verifyNot checkBashisms "#!/bin/sh\necho -Q foo" prop_checkBashisms78 = verify checkBashisms "#!/bin/sh\necho -ne foo" +prop_checkBashisms79 = verify checkBashisms "#!/bin/sh\nhash -l" +prop_checkBashisms80 = verifyNot checkBashisms "#!/bin/sh\nhash -r" +prop_checkBashisms81 = verifyNot checkBashisms "#!/bin/dash\nhash -v" checkBashisms = ForShell [Sh, Dash] $ \t -> do params <- ask kludge params t @@ -307,6 +310,7 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do ("cd", Just ["L", "P"]), ("exec", Just []), ("export", Just ["p"]), + ("hash", Just $ if isDash then ["r", "v"] else ["r"]), ("jobs", Just ["l", "p"]), ("printf", Just []), ("read", Just $ if isDash then ["r", "p"] else ["r"]), From 2ea22931545ce7251bfdc2f61e04153bfccfa9f6 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 27 Jan 2019 15:01:52 -0800 Subject: [PATCH 083/763] Update SC1008 to suggest using directive. --- src/ShellCheck/Parser.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 14e31e3..8a7f794 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -3032,7 +3032,7 @@ readScriptFile = do case isValidShell s of Just True -> return () Just False -> parseProblemAt pos ErrorC 1071 "ShellCheck only supports sh/bash/dash/ksh scripts. Sorry!" - Nothing -> parseProblemAt pos InfoC 1008 "This shebang was unrecognized. Note that ShellCheck only handles sh/bash/dash/ksh." + Nothing -> parseProblemAt pos ErrorC 1008 "This shebang was unrecognized. ShellCheck only supports sh/bash/dash/ksh. Add a 'shell' directive to specify." isValidShell s = let good = s == "" || any (`isPrefixOf` s) goodShells From acef53be9c199bc31654cb298ca07a272d83d6a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristian=20Adri=C3=A1n=20Ontivero?= Date: Mon, 28 Jan 2019 19:44:09 +0100 Subject: [PATCH 084/763] Check set flags under dash & POSIX sh (fixes #990) The set builtin accepts certain flags, and some longer synonyms (for instance set -e is the same as set -o errexit) under POSIX sh. This makes ShellCheck warn if any of the used flags and options are undefined when targeting POSIX sh. This fixes #990, while adding general flag-support checking for set in the process. --- src/ShellCheck/Checks/ShellSupport.hs | 54 +++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 54af521..3de5dbd 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -33,6 +33,7 @@ import Data.Char import Data.List import Data.Maybe import qualified Data.Map as Map +import qualified Data.Set as Set import Test.QuickCheck.All (forAllProperties) import Test.QuickCheck.Test (quickCheckWithResult, stdArgs, maxSuccess) @@ -161,6 +162,14 @@ prop_checkBashisms78 = verify checkBashisms "#!/bin/sh\necho -ne foo" prop_checkBashisms79 = verify checkBashisms "#!/bin/sh\nhash -l" prop_checkBashisms80 = verifyNot checkBashisms "#!/bin/sh\nhash -r" prop_checkBashisms81 = verifyNot checkBashisms "#!/bin/dash\nhash -v" +prop_checkBashisms82 = verifyNot checkBashisms "#!/bin/sh\nset -v +o allexport -o errexit -C" +prop_checkBashisms83 = verifyNot checkBashisms "#!/bin/sh\nset --" +prop_checkBashisms84 = verify checkBashisms "#!/bin/sh\nset -o pipefail" +prop_checkBashisms85 = verify checkBashisms "#!/bin/sh\nset -B" +prop_checkBashisms86 = verifyNot checkBashisms "#!/bin/dash\nset -o emacs" +prop_checkBashisms87 = verify checkBashisms "#!/bin/sh\nset -o emacs" +prop_checkBashisms88 = verifyNot checkBashisms "#!/bin/sh\nset -- wget -o foo 'https://some.url'" +prop_checkBashisms89 = verifyNot checkBashisms "#!/bin/sh\nopts=$-\nset -\"$opts\"" checkBashisms = ForShell [Sh, Dash] $ \t -> do params <- ask kludge params t @@ -263,6 +272,51 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do warnMsg (getId arg) "exec flags are" bashism t@(T_SimpleCommand id _ _) | t `isCommand` "let" = warnMsg id "'let' is" + bashism t@(T_SimpleCommand _ _ (cmd:arg:rest)) + | t `isCommand` "set" = unless isDash $ checkOptions (arg:rest) + where + -- Check a flag-option pair (such as -o errexit) + checkOptions (flag:opt:rest) + | flag' `matches` oFlagRegex = do + when (opt' `notElem` longOptions) $ + warnMsg (getId opt) $ "set option " <> opt' <> " is" + checkFlags (flag:rest) + | otherwise = checkFlags (flag:opt:rest) + where + flag' = concat $ getLiteralString flag + opt' = concat $ getLiteralString opt + checkOptions (flag:rest) = checkFlags (flag:rest) + checkOptions _ = return () + + -- Check that each option in a sequence of flags + -- (such as -aveo) is valid + checkFlags (flag:rest) + | startsOption flag' = do + unless (flag' `matches` validFlagsRegex) $ + forM_ (tail flag') $ \letter -> + when (letter `notElem` optionsSet) $ + warnMsg (getId flag) $ "set flag " <> ('-':letter:" is") + checkOptions rest + | beginsWithDoubleDash flag' = do + warnMsg (getId flag) $ "set flag " <> flag' <> " is" + checkOptions rest + -- Either a word that doesn't start with a dash, or simply '--', + -- so stop checking. + | otherwise = return () + where + flag' = concat $ getLiteralString flag + checkFlags [] = return () + + options = "abCefhmnuvxo" + optionsSet = Set.fromList options + startsOption = (`matches` mkRegex "^(\\+|-[^-])") + oFlagRegex = mkRegex $ "^[-+][" <> options <> "]*o$" + validFlagsRegex = mkRegex $ "^[-+]([" <> options <> "]+o?|o)$" + beginsWithDoubleDash = (`matches` mkRegex "^--.+$") + longOptions = Set.fromList + [ "allexport", "errexit", "ignoreeof", "monitor", "noclobber" + , "noexec", "noglob", "nolog", "notify" , "nounset", "verbose" + , "vi", "xtrace" ] bashism t@(T_SimpleCommand id _ (cmd:rest)) = let name = fromMaybe "" $ getCommandName t From d984f8cbe71c72944bde2dbf1479112ce7463909 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Fri, 8 Feb 2019 22:36:22 -0800 Subject: [PATCH 085/763] Don't look at 'set' options after a non-literal. --- src/ShellCheck/Checks/ShellSupport.hs | 29 ++++++++++++++++----------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 3de5dbd..9b65ce7 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -170,6 +170,7 @@ prop_checkBashisms86 = verifyNot checkBashisms "#!/bin/dash\nset -o emacs" prop_checkBashisms87 = verify checkBashisms "#!/bin/sh\nset -o emacs" prop_checkBashisms88 = verifyNot checkBashisms "#!/bin/sh\nset -- wget -o foo 'https://some.url'" prop_checkBashisms89 = verifyNot checkBashisms "#!/bin/sh\nopts=$-\nset -\"$opts\"" +prop_checkBashisms90 = verifyNot checkBashisms "#!/bin/sh\nset -o \"$opt\"" checkBashisms = ForShell [Sh, Dash] $ \t -> do params <- ask kludge params t @@ -272,39 +273,43 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do warnMsg (getId arg) "exec flags are" bashism t@(T_SimpleCommand id _ _) | t `isCommand` "let" = warnMsg id "'let' is" - bashism t@(T_SimpleCommand _ _ (cmd:arg:rest)) - | t `isCommand` "set" = unless isDash $ checkOptions (arg:rest) + bashism t@(T_SimpleCommand _ _ (cmd:args)) + | t `isCommand` "set" = unless isDash $ + checkOptions $ getLiteralArgs args where + -- Get the literal options from a list of arguments, + -- up until the first non-literal one + getLiteralArgs :: [Token] -> [(Id, String)] + getLiteralArgs (first:rest) = fromMaybe [] $ do + str <- getLiteralString first + return $ (getId first, str) : getLiteralArgs rest + getLiteralArgs [] = [] + -- Check a flag-option pair (such as -o errexit) - checkOptions (flag:opt:rest) + checkOptions (flag@(fid,flag') : opt@(oid,opt') : rest) | flag' `matches` oFlagRegex = do when (opt' `notElem` longOptions) $ - warnMsg (getId opt) $ "set option " <> opt' <> " is" + warnMsg oid $ "set option " <> opt' <> " is" checkFlags (flag:rest) | otherwise = checkFlags (flag:opt:rest) - where - flag' = concat $ getLiteralString flag - opt' = concat $ getLiteralString opt checkOptions (flag:rest) = checkFlags (flag:rest) checkOptions _ = return () -- Check that each option in a sequence of flags -- (such as -aveo) is valid - checkFlags (flag:rest) + checkFlags (flag@(fid, flag'):rest) | startsOption flag' = do unless (flag' `matches` validFlagsRegex) $ forM_ (tail flag') $ \letter -> when (letter `notElem` optionsSet) $ - warnMsg (getId flag) $ "set flag " <> ('-':letter:" is") + warnMsg fid $ "set flag " <> ('-':letter:" is") checkOptions rest | beginsWithDoubleDash flag' = do - warnMsg (getId flag) $ "set flag " <> flag' <> " is" + warnMsg fid $ "set flag " <> flag' <> " is" checkOptions rest -- Either a word that doesn't start with a dash, or simply '--', -- so stop checking. | otherwise = return () - where - flag' = concat $ getLiteralString flag checkFlags [] = return () options = "abCefhmnuvxo" From d3f6e045e2e80eb350650a77734b047927ee7db1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristian=20Adri=C3=A1n=20Ontivero?= Date: Wed, 30 Jan 2019 11:26:08 +0100 Subject: [PATCH 086/763] Check wait flags in dash & POSIX sh Flags for the wait builtin are undefined under both POSIX sh and dash. Bash though, accepts [-fn]. --- src/ShellCheck/Checks/ShellSupport.hs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 9b65ce7..de9efc8 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -171,6 +171,7 @@ prop_checkBashisms87 = verify checkBashisms "#!/bin/sh\nset -o emacs" prop_checkBashisms88 = verifyNot checkBashisms "#!/bin/sh\nset -- wget -o foo 'https://some.url'" prop_checkBashisms89 = verifyNot checkBashisms "#!/bin/sh\nopts=$-\nset -\"$opts\"" prop_checkBashisms90 = verifyNot checkBashisms "#!/bin/sh\nset -o \"$opt\"" +prop_checkBashisms91 = verify checkBashisms "#!/bin/sh\nwait -n" checkBashisms = ForShell [Sh, Dash] $ \t -> do params <- ask kludge params t @@ -378,7 +379,8 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do ("type", Just []), ("ulimit", if isDash then Nothing else Just ["f"]), ("umask", Just ["S"]), - ("unset", Just ["f", "v"]) + ("unset", Just ["f", "v"]), + ("wait", Just []) ] bashism t@(T_SourceCommand id src _) = let name = fromMaybe "" $ getCommandName src From 3a276bd33657eea36d3912c18b6a764dd0f2e8b1 Mon Sep 17 00:00:00 2001 From: cclauss Date: Mon, 18 Feb 2019 09:36:36 +0100 Subject: [PATCH 087/763] README.md: pipe wget | tar to reduce duplication --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 597be37..ecd48c4 100644 --- a/README.md +++ b/README.md @@ -191,8 +191,7 @@ To install it on Redhat/Fedora/CentOS, run `yum -y install xz`. ```bash export scversion="stable" # or "v0.4.7", or "latest" -wget "https://storage.googleapis.com/shellcheck/shellcheck-${scversion}.linux.x86_64.tar.xz" -tar --xz -xvf shellcheck-"${scversion}".linux.x86_64.tar.xz +wget -qO- "https://storage.googleapis.com/shellcheck/shellcheck-stable.linux.x86_64.tar.xz" | tar -xJv cp shellcheck-"${scversion}"/shellcheck /usr/bin/ shellcheck --version ``` From d31d31df231b27b659641be556179bd8277ac524 Mon Sep 17 00:00:00 2001 From: cclauss Date: Mon, 18 Feb 2019 10:18:21 +0100 Subject: [PATCH 088/763] wget -qO- "https://storage.googleapis.com/shellcheck/shellcheck-"${scversion}".linux.x86_64.tar.xz" | tar -xJv --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ecd48c4..38d6b3b 100644 --- a/README.md +++ b/README.md @@ -191,7 +191,7 @@ To install it on Redhat/Fedora/CentOS, run `yum -y install xz`. ```bash export scversion="stable" # or "v0.4.7", or "latest" -wget -qO- "https://storage.googleapis.com/shellcheck/shellcheck-stable.linux.x86_64.tar.xz" | tar -xJv +wget -qO- "https://storage.googleapis.com/shellcheck/shellcheck-"${scversion}".linux.x86_64.tar.xz" | tar -xJv cp shellcheck-"${scversion}"/shellcheck /usr/bin/ shellcheck --version ``` From bd19ab4fa9c309ec623dbf2cdbd4844d2934917f Mon Sep 17 00:00:00 2001 From: Tito Sacchi Date: Sun, 24 Feb 2019 09:45:31 +0100 Subject: [PATCH 089/763] Fix issues #896 and #433: printf -v and arrays --- src/ShellCheck/Analytics.hs | 1 + src/ShellCheck/AnalyzerLib.hs | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 85b8bc6..7498bb8 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1954,6 +1954,7 @@ prop_checkUnassignedReferences33= verifyNotTree checkUnassignedReferences "f() { prop_checkUnassignedReferences34= verifyNotTree checkUnassignedReferences "declare -A foo; (( foo[bar] ))" prop_checkUnassignedReferences35= verifyNotTree checkUnassignedReferences "echo ${arr[foo-bar]:?fail}" prop_checkUnassignedReferences36= verifyNotTree checkUnassignedReferences "read -a foo -r <<<\"foo bar\"; echo \"$foo\"" +prop_checkUnassignedReferences37= verifyNotTree checkUnassignedReferences "var=howdy; printf -v 'array[0]' %s \"$var\"; printf %s \"${array[0]}\";" checkUnassignedReferences params t = warnings where (readMap, writeMap) = execState (mapM tally $ variableFlow params) (Map.empty, Map.empty) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index e2cec40..01c87e4 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -649,7 +649,11 @@ getModifiedVariableCommand base@(T_SimpleCommand _ _ (T_NormalWord _ (T_Literal getPrintfVariable list = f $ map (\x -> (x, getLiteralString x)) list where - f ((_, Just "-v") : (t, Just var) : _) = return (base, t, var, DataString $ SourceFrom list) + f ((_, Just "-v") : (t, Just var) : _) = return (base, t, varName, varType $ SourceFrom list) + where + (varName, varType) = case elemIndex '[' var of + Just i -> (take i var, DataArray) + Nothing -> (var, DataString) f (_:rest) = f rest f [] = fail "not found" From e45d81c8fa9be3ba376fae9f95477a82fa824d5f Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 2 Mar 2019 13:36:55 -0800 Subject: [PATCH 090/763] Update README.md with more CI and build info --- README.md | 40 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 38d6b3b..d426216 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,6 @@ See [the gallery of bad code](README.md#user-content-gallery-of-bad-code) for ex - [In your editor](#in-your-editor) - [In your build or test suites](#in-your-build-or-test-suites) - [Installing](#installing) -- [Travis CI](#travis-ci) - [Compiling from source](#compiling-from-source) - [Installing Cabal](#installing-cabal) - [Compiling ShellCheck](#compiling-shellcheck) @@ -85,8 +84,45 @@ You can see ShellCheck suggestions directly in a variety of editors. ### In your build or test suites While ShellCheck is mostly intended for interactive use, it can easily be added to builds or test suites. +It makes canonical use of exit codes, so you can just add a `shellcheck` command as part of the process. -ShellCheck makes canonical use of exit codes, and can output simple JSON, CheckStyle compatible XML, GCC compatible warnings as well as human readable text (with or without ANSI colors). See the [Integration](https://github.com/koalaman/shellcheck/wiki/Integration) wiki page for more documentation. +For example, in a Makefile: + +``` +check-scripts: + # Fail if any of these files have warnings + shellcheck myscripts/*.sh +``` + +or in a Travis CI `.travis.yml` file: + +``` +script: + # Fail if any of these files have warnings + - shellcheck myscripts/*.sh +``` + +Services and platforms that have ShellCheck pre-installed and ready to use: + +* [Travis CI](https://travis-ci.org/) +* [Codacy](https://www.codacy.com/) +* [Code Climate](https://codeclimate.com/) +* [Code Factor](https://www.codefactor.io/) + +Services and platforms with third party plugins: + +* [SonarQube](https://www.sonarqube.org/) through [sonar-shellcheck-plugin](https://github.com/emerald-squad/sonar-shellcheck-plugin) + +Most other services, including [GitLab](https://about.gitlab.com/), let you install +ShellCheck yourself, either through the system's package manager (see [Installing](#installing)), +or by downloading and unpacking a [binary release](#installing-the-shellcheck-binary). + +It's a good idea to manually install a specific ShellCheck version regardless. This avoids +any surprise build breaks when a new version with new warnings is published. + +For customized filtering or reporting, ShellCheck can output simple JSON, CheckStyle compatible XML, +GCC compatible warnings as well as human readable text (with or without ANSI colors). See the +[Integration](https://github.com/koalaman/shellcheck/wiki/Integration) wiki page for more documentation. ## Installing From 25ea4054681e4969f2ed25b63c4b987698df09da Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 2 Mar 2019 13:49:21 -0800 Subject: [PATCH 091/763] Fix typo in man page (fixes #1486) --- shellcheck.1.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shellcheck.1.md b/shellcheck.1.md index 3e026cb..36c341a 100644 --- a/shellcheck.1.md +++ b/shellcheck.1.md @@ -36,7 +36,7 @@ not warn at all, as `ksh` supports decimals in arithmetic contexts. : Emit warnings in sourced files. Normally, `shellcheck` will only warn about issues in the specified files. With this option, any issues in - sourced files files will also be reported. + sourced files will also be reported. **-C**[*WHEN*],\ **--color**[=*WHEN*] From 293c3b27b81053d7c20cedffc95523ca35bb0033 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 3 Mar 2019 13:37:32 -0800 Subject: [PATCH 092/763] Continue on parse errors in backticks (fixes #1475) --- src/ShellCheck/Parser.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 8a7f794..47ea8c9 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -1200,7 +1200,7 @@ readBackTicked quoted = called "backtick expansion" $ do suggestForgotClosingQuote startPos endPos "backtick expansion" -- Result positions may be off due to escapes - result <- subParse subStart subParser (unEscape subString) + result <- subParse subStart (tryWithErrors subParser <|> return []) (unEscape subString) return $ T_Backticked id result where unEscape [] = [] From 581bcc3907ab98e919a7dd60566810a928c46b95 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 3 Mar 2019 18:53:43 -0800 Subject: [PATCH 093/763] Add support for `.shellcheckrc` files --- CHANGELOG.md | 1 + ShellCheck.cabal | 14 ++++--- shellcheck.1.md | 29 ++++++++++++++ shellcheck.hs | 72 +++++++++++++++++++++++++++++++-- src/ShellCheck/Checker.hs | 33 ++++++++++++++++ src/ShellCheck/Interface.hs | 22 ++++++++--- src/ShellCheck/Parser.hs | 79 ++++++++++++++++++++++++++++++++----- 7 files changed, 226 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c6f54c..586ede6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ### Added - Preliminary support for fix suggestions - Files containing Bats tests can now be checked +- Directory wide directives can now be placed in a `.shellcheckrc` - SC2246: Warn if a shebang's interpreter ends with / - SC2245: Warn that Ksh ignores all but the first glob result in `[` - SC2243/SC2244: Suggest using explicit -n for `[ $foo ]` diff --git a/ShellCheck.cabal b/ShellCheck.cabal index ff612ed..8e3ddb5 100644 --- a/ShellCheck.cabal +++ b/ShellCheck.cabal @@ -96,14 +96,15 @@ executable shellcheck array, base >= 4 && < 5, bytestring, - deepseq >= 1.4.0.0, - ShellCheck, containers, + deepseq >= 1.4.0.0, directory, mtl >= 2.2.1, + filepath, parsec >= 3.0, QuickCheck >= 2.7.4, - regex-tdfa + regex-tdfa, + ShellCheck main-is: shellcheck.hs test-suite test-shellcheck @@ -113,13 +114,14 @@ test-suite test-shellcheck array, base >= 4 && < 5, bytestring, - deepseq >= 1.4.0.0, - ShellCheck, containers, + deepseq >= 1.4.0.0, directory, mtl >= 2.2.1, + filepath, parsec, QuickCheck >= 2.7.4, - regex-tdfa + regex-tdfa, + ShellCheck main-is: test/shellcheck.hs diff --git a/shellcheck.1.md b/shellcheck.1.md index 36c341a..739628a 100644 --- a/shellcheck.1.md +++ b/shellcheck.1.md @@ -63,6 +63,10 @@ not warn at all, as `ksh` supports decimals in arithmetic contexts. standard output. Subsequent **-f** options are ignored, see **FORMATS** below for more information. +**--norc** + +: Don't try to look for .shellcheckrc configuration files. + **-S**\ *SEVERITY*,\ **--severity=***severity* : Specify minimum severity of errors to consider. Valid values are *error*, @@ -192,6 +196,31 @@ Valid keys are: files meant to be included (and thus lacking a shebang), or possibly as a more targeted alternative to 'disable=2039'. +# RC FILES +Unless `--norc` is used, ShellCheck will look for a file `.shellcheckrc` or +`shellcheckrc` in the script's directory and each parent directory. If found, +it will read `key=value` pairs from it and treat them as file-wide directives. + +Here is an example `.shellcheckrc`: + + # Don't suggest using -n in [ $var ] + disable=SC2244 + + # Allow using `which` since it gives full paths and is common enough + disable=SC2230 + +If no `.shellcheckrc` is found in any of the parent directories, ShellCheck +will look in `~/.shellcheckrc` followed by the XDG config directory +(usually `~/.config/shellcheckrc`) on Unix, or %APPDATA%/shellcheckrc` on +Windows. Only the first file found will be used. + +Note for Snap users: the Snap sandbox disallows access to hidden files. +Use `shellcheckrc` without the dot instead. + +Note for Docker users: ShellCheck will only be able to look for files that +are mounted in the container, so `~/.shellcheckrc` will not be read. + + # ENVIRONMENT VARIABLES The environment variable `SHELLCHECK_OPTS` can be set with default flags: diff --git a/shellcheck.hs b/shellcheck.hs index a93da2a..8887a85 100644 --- a/shellcheck.hs +++ b/shellcheck.hs @@ -47,6 +47,7 @@ import System.Console.GetOpt import System.Directory import System.Environment import System.Exit +import System.FilePath import System.IO data Flag = Flag String String @@ -95,6 +96,8 @@ options = [ Option "f" ["format"] (ReqArg (Flag "format") "FORMAT") $ "Output format (" ++ formatList ++ ")", + Option "" ["norc"] + (NoArg $ Flag "norc" "true") "Don't look for .shellcheckrc files", Option "s" ["shell"] (ReqArg (Flag "shell") "SHELLNAME") "Specify dialect (sh, bash, dash, ksh)", @@ -330,7 +333,16 @@ parseOption flag options = } } - _ -> return options + Flag "norc" _ -> + return options { + checkSpec = (checkSpec options) { + csIgnoreRC = True + } + } + + Flag str _ -> do + printErr $ "Internal error for --" ++ str ++ ". Please file a bug :(" + return options where die s = do printErr s @@ -345,12 +357,15 @@ parseOption flag options = ioInterface options files = do inputs <- mapM normalize files cache <- newIORef emptyCache + configCache <- newIORef ("", Nothing) return SystemInterface { - siReadFile = get cache inputs + siReadFile = get cache inputs, + siGetConfig = getConfig configCache } where emptyCache :: Map.Map FilePath String emptyCache = Map.empty + get cache inputs file = do map <- readIORef cache case Map.lookup file map of @@ -367,7 +382,6 @@ ioInterface options files = do return $ Right contents ) `catch` handler else return $ Left (file ++ " was not specified as input (see shellcheck -x).") - where handler :: IOException -> IO (Either ErrorMessage String) handler ex = return . Left $ show ex @@ -385,6 +399,58 @@ ioInterface options files = do fallback :: FilePath -> IOException -> IO FilePath fallback path _ = return path + -- Returns the name and contents of .shellcheckrc for the given file + getConfig cache filename = do + path <- normalize filename + let dir = takeDirectory path + (previousPath, result) <- readIORef cache + if dir == previousPath + then return result + else do + paths <- getConfigPaths dir + result <- findConfig paths + writeIORef cache (dir, result) + return result + + findConfig paths = + case paths of + (file:rest) -> do + contents <- readConfig file + if isJust contents + then return contents + else findConfig rest + [] -> return Nothing + + -- Get a list of candidate filenames. This includes .shellcheckrc + -- in all parent directories, plus the user's home dir and xdg dir. + -- The dot is optional for Windows and Snap users. + getConfigPaths dir = do + let next = takeDirectory dir + rest <- if next /= dir + then getConfigPaths next + else defaultPaths `catch` + ((const $ return []) :: IOException -> IO [FilePath]) + return $ (dir ".shellcheckrc") : (dir "shellcheckrc") : rest + + defaultPaths = do + home <- getAppUserDataDirectory "shellcheckrc" + xdg <- getXdgDirectory XdgConfig "shellcheckrc" + return [home, xdg] + + readConfig file = do + exists <- doesPathExist file + if exists + then do + (contents, _) <- inputFile file `catch` handler file + return $ Just (file, contents) + else + return Nothing + where + handler :: FilePath -> IOException -> IO (String, Bool) + handler file err = do + putStrLn $ file ++ ": " ++ show err + return ("", True) + inputFile file = do (handle, shouldCache) <- if file == "-" diff --git a/src/ShellCheck/Checker.hs b/src/ShellCheck/Checker.hs index f9c1ede..8b3eb76 100644 --- a/src/ShellCheck/Checker.hs +++ b/src/ShellCheck/Checker.hs @@ -72,6 +72,7 @@ checkScript sys spec = do psFilename = csFilename spec, psScript = contents, psCheckSourced = csCheckSourced spec, + psIgnoreRC = csIgnoreRC spec, psShellTypeOverride = csShellTypeOverride spec } let parseMessages = prComments result @@ -146,6 +147,9 @@ checkOptionIncludes includes src = csCheckSourced = True } +checkWithRc rc = getErrors + (mockRcFile rc $ mockedSystemInterface []) + prop_findsParseIssue = check "echo \"$12\"" == [1037] prop_commentDisablesParseIssue1 = @@ -299,5 +303,34 @@ prop_optionIncludes4 = -- expect 2086 & 2154, only 2154 included, so only that's reported [2154] == checkOptionIncludes (Just [2154]) "#!/bin/sh\n var='a b'\n echo $var\n echo $bar" + +prop_readsRcFile = result == [] + where + result = checkWithRc "disable=2086" emptyCheckSpec { + csScript = "#!/bin/sh\necho $1", + csIgnoreRC = False + } + +prop_canUseNoRC = result == [2086] + where + result = checkWithRc "disable=2086" emptyCheckSpec { + csScript = "#!/bin/sh\necho $1", + csIgnoreRC = True + } + +prop_NoRCWontLookAtFile = result == [2086] + where + result = checkWithRc (error "Fail") emptyCheckSpec { + csScript = "#!/bin/sh\necho $1", + csIgnoreRC = True + } + +prop_brokenRcGetsWarning = result == [1134, 2086] + where + result = checkWithRc "rofl" emptyCheckSpec { + csScript = "#!/bin/sh\necho $1", + csIgnoreRC = False + } + return [] runTests = $quickCheckAll diff --git a/src/ShellCheck/Interface.hs b/src/ShellCheck/Interface.hs index b492b3f..3a6d685 100644 --- a/src/ShellCheck/Interface.hs +++ b/src/ShellCheck/Interface.hs @@ -21,9 +21,9 @@ module ShellCheck.Interface ( SystemInterface(..) - , CheckSpec(csFilename, csScript, csCheckSourced, csIncludedWarnings, csExcludedWarnings, csShellTypeOverride, csMinSeverity) + , CheckSpec(csFilename, csScript, csCheckSourced, csIncludedWarnings, csExcludedWarnings, csShellTypeOverride, csMinSeverity, csIgnoreRC) , CheckResult(crFilename, crComments) - , ParseSpec(psFilename, psScript, psCheckSourced, psShellTypeOverride) + , ParseSpec(psFilename, psScript, psCheckSourced, psIgnoreRC, psShellTypeOverride) , ParseResult(prComments, prTokenPositions, prRoot) , AnalysisSpec(asScript, asShellType, asFallbackShell, asExecutionMode, asCheckSourced, asTokenPositions) , AnalysisResult(arComments) @@ -46,6 +46,7 @@ module ShellCheck.Interface , newPosition , newTokenComment , mockedSystemInterface + , mockRcFile , newParseSpec , emptyCheckSpec , newPositionedComment @@ -69,9 +70,11 @@ import GHC.Generics (Generic) import qualified Data.Map as Map -newtype SystemInterface m = SystemInterface { +data SystemInterface m = SystemInterface { -- Read a file by filename, or return an error - siReadFile :: String -> m (Either ErrorMessage String) + siReadFile :: String -> m (Either ErrorMessage String), + -- Get the configuration file (name, contents) for a filename + siGetConfig :: String -> m (Maybe (FilePath, String)) } -- ShellCheck input and output @@ -79,6 +82,7 @@ data CheckSpec = CheckSpec { csFilename :: String, csScript :: String, csCheckSourced :: Bool, + csIgnoreRC :: Bool, csExcludedWarnings :: [Integer], csIncludedWarnings :: Maybe [Integer], csShellTypeOverride :: Maybe Shell, @@ -101,6 +105,7 @@ emptyCheckSpec = CheckSpec { csFilename = "", csScript = "", csCheckSourced = False, + csIgnoreRC = False, csExcludedWarnings = [], csIncludedWarnings = Nothing, csShellTypeOverride = Nothing, @@ -112,6 +117,7 @@ newParseSpec = ParseSpec { psFilename = "", psScript = "", psCheckSourced = False, + psIgnoreRC = False, psShellTypeOverride = Nothing } @@ -120,6 +126,7 @@ data ParseSpec = ParseSpec { psFilename :: String, psScript :: String, psCheckSourced :: Bool, + psIgnoreRC :: Bool, psShellTypeOverride :: Maybe Shell } deriving (Show, Eq) @@ -279,7 +286,8 @@ data ColorOption = -- For testing mockedSystemInterface :: [(String, String)] -> SystemInterface Identity mockedSystemInterface files = SystemInterface { - siReadFile = rf + siReadFile = rf, + siGetConfig = const $ return Nothing } where rf file = @@ -287,3 +295,7 @@ mockedSystemInterface files = SystemInterface { [] -> return $ Left "File not included in mock." [(_, contents)] -> return $ Right contents +mockRcFile rcfile mock = mock { + siGetConfig = const . return $ Just (".shellcheckrc", rcfile) +} + diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 47ea8c9..0289e0d 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -113,6 +113,7 @@ allspacing = do allspacingOrFail = do s <- allspacing when (null s) $ fail "Expected whitespace" + return s readUnicodeQuote = do start <- startSpan @@ -306,6 +307,8 @@ initialSystemState = SystemState { data Environment m = Environment { systemInterface :: SystemInterface m, checkSourced :: Bool, + ignoreRC :: Bool, + currentFilename :: String, shellTypeOverride :: Maybe Shell } @@ -949,9 +952,12 @@ prop_readAnnotation6 = isOk readAnnotation "# shellcheck disable=SC1234 # shellc readAnnotation = called "shellcheck directive" $ do try readAnnotationPrefix many1 linewhitespace + readAnnotationWithoutPrefix + +readAnnotationWithoutPrefix = do values <- many1 readKey optional readAnyComment - void linefeed <|> do + void linefeed <|> eof <|> do parseNote ErrorC 1125 "Invalid key=value pair? Ignoring the rest of this directive starting here." many (noneOf "\n") void linefeed <|> eof @@ -2104,7 +2110,7 @@ readSource t@(T_Redirecting _ _ (T_SimpleCommand cmdId _ (cmd:file:_))) = do subRead name script = withContext (ContextSource name) $ inSeparateContext $ - subParse (initialPos name) readScript script + subParse (initialPos name) (readScriptFile True) script readSource t = return t @@ -2980,12 +2986,55 @@ verifyEof = eof <|> choice [ try (lookAhead p) action -prop_readScript1 = isOk readScriptFile "#!/bin/bash\necho hello world\n" -prop_readScript2 = isWarning readScriptFile "#!/bin/bash\r\necho hello world\n" -prop_readScript3 = isWarning readScriptFile "#!/bin/bash\necho hello\xA0world" -prop_readScript4 = isWarning readScriptFile "#!/usr/bin/perl\nfoo=(" -prop_readScript5 = isOk readScriptFile "#!/bin/bash\n#This is an empty script\n\n" -readScriptFile = do + +readConfigFile :: Monad m => FilePath -> SCParser m [Annotation] +readConfigFile filename = do + shouldIgnore <- Mr.asks ignoreRC + if shouldIgnore then return [] else read' filename + where + read' filename = do + sys <- Mr.asks systemInterface + contents <- system $ siGetConfig sys filename + case contents of + Nothing -> return [] + Just (file, str) -> readConfig file str + + readConfig filename contents = do + result <- lift $ runParserT readConfigKVs initialUserState filename contents + case result of + Right result -> + return result + + Left err -> do + parseProblem ErrorC 1134 $ errorFor filename err + return [] + + errorFor filename err = + let line = "line " ++ (show . sourceLine $ errorPos err) + suggestion = getStringFromParsec $ errorMessages err + in + "Failed to process " ++ filename ++ ", " ++ line ++ ": " + ++ suggestion + +prop_readConfigKVs1 = isOk readConfigKVs "disable=1234" +prop_readConfigKVs2 = isOk readConfigKVs "# Comment\ndisable=1234 # Comment\n" +prop_readConfigKVs3 = isOk readConfigKVs "" +prop_readConfigKVs4 = isOk readConfigKVs "\n\n\n\n\t \n" +prop_readConfigKVs5 = isOk readConfigKVs "# shellcheck accepts annotation-like comments in rc files\ndisable=1234" +readConfigKVs = do + anySpacingOrComment + annotations <- many (readAnnotationWithoutPrefix <* anySpacingOrComment) + eof + return $ concat annotations +anySpacingOrComment = + many (void allspacingOrFail <|> void readAnyComment) + +prop_readScript1 = isOk readScript "#!/bin/bash\necho hello world\n" +prop_readScript2 = isWarning readScript "#!/bin/bash\r\necho hello world\n" +prop_readScript3 = isWarning readScript "#!/bin/bash\necho hello\xA0world" +prop_readScript4 = isWarning readScript "#!/usr/bin/perl\nfoo=(" +prop_readScript5 = isOk readScript "#!/bin/bash\n#This is an empty script\n\n" +readScriptFile sourced = do start <- startSpan pos <- getPosition optional $ do @@ -2995,7 +3044,13 @@ readScriptFile = do sb <- option "" readShebang allspacing annotationStart <- startSpan - annotations <- readAnnotations + fileAnnotations <- readAnnotations + rcAnnotations <- if sourced + then return [] + else do + filename <- Mr.asks currentFilename + readConfigFile filename + let annotations = fileAnnotations ++ rcAnnotations annotationId <- endSpan annotationStart let shellAnnotationSpecified = any (\x -> case x of ShellOverride {} -> True; _ -> False) annotations @@ -3065,7 +3120,7 @@ readScriptFile = do readUtf8Bom = called "Byte Order Mark" $ string "\xFEFF" -readScript = readScriptFile +readScript = readScriptFile False -- Interactively run a specific parser in ghci: -- debugParse readSimpleCommand "echo 'hello world'" @@ -3100,6 +3155,8 @@ testEnvironment = Environment { systemInterface = (mockedSystemInterface []), checkSourced = False, + currentFilename = "myscript", + ignoreRC = False, shellTypeOverride = Nothing } @@ -3275,6 +3332,8 @@ parseScript sys spec = env = Environment { systemInterface = sys, checkSourced = psCheckSourced spec, + currentFilename = psFilename spec, + ignoreRC = psIgnoreRC spec, shellTypeOverride = psShellTypeOverride spec } From 4dfb3fce9c11f718f54f8908490cf8ee1290dd94 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 3 Mar 2019 19:00:09 -0800 Subject: [PATCH 094/763] Add missing backtick in man page --- shellcheck.1.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shellcheck.1.md b/shellcheck.1.md index 739628a..1499c80 100644 --- a/shellcheck.1.md +++ b/shellcheck.1.md @@ -211,7 +211,7 @@ Here is an example `.shellcheckrc`: If no `.shellcheckrc` is found in any of the parent directories, ShellCheck will look in `~/.shellcheckrc` followed by the XDG config directory -(usually `~/.config/shellcheckrc`) on Unix, or %APPDATA%/shellcheckrc` on +(usually `~/.config/shellcheckrc`) on Unix, or `%APPDATA%/shellcheckrc` on Windows. Only the first file found will be used. Note for Snap users: the Snap sandbox disallows access to hidden files. From bbe5155e63c5bf942f55c4b15467b29245b556d6 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 4 Mar 2019 18:18:58 -0800 Subject: [PATCH 095/763] Use less modern APIs to support more GHC versions --- shellcheck.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shellcheck.hs b/shellcheck.hs index 8887a85..446aa3b 100644 --- a/shellcheck.hs +++ b/shellcheck.hs @@ -438,7 +438,7 @@ ioInterface options files = do return [home, xdg] readConfig file = do - exists <- doesPathExist file + exists <- doesFileExist file if exists then do (contents, _) <- inputFile file `catch` handler file From ed92fe501f81bdc707ec5caa64482814c020fc89 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 6 Mar 2019 17:42:55 -0800 Subject: [PATCH 096/763] Fix internal error for --format (fixes #1507) --- shellcheck.hs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/shellcheck.hs b/shellcheck.hs index 446aa3b..483da1b 100644 --- a/shellcheck.hs +++ b/shellcheck.hs @@ -340,6 +340,9 @@ parseOption flag options = } } + -- This flag is handled specially in 'process' + Flag "format" _ -> return options + Flag str _ -> do printErr $ "Internal error for --" ++ str ++ ". Please file a bug :(" return options From b456987b847f91ca7b23a3cb7d1cc457bde84816 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 6 Mar 2019 19:19:00 -0800 Subject: [PATCH 097/763] Add the minimum version of 'directory' --- ShellCheck.cabal | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ShellCheck.cabal b/ShellCheck.cabal index 8e3ddb5..099052e 100644 --- a/ShellCheck.cabal +++ b/ShellCheck.cabal @@ -57,8 +57,9 @@ library bytestring, containers >= 0.5, deepseq >= 1.4.0.0, - directory, + directory >= 1.2.3.0, mtl >= 2.2.1, + filepath, parsec, regex-tdfa, QuickCheck >= 2.7.4, @@ -98,7 +99,7 @@ executable shellcheck bytestring, containers, deepseq >= 1.4.0.0, - directory, + directory >= 1.2.3.0, mtl >= 2.2.1, filepath, parsec >= 3.0, @@ -116,7 +117,7 @@ test-suite test-shellcheck bytestring, containers, deepseq >= 1.4.0.0, - directory, + directory >= 1.2.3.0, mtl >= 2.2.1, filepath, parsec, From c53c8a5eaddecf78937850d41d5b467ff5ab2358 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 17 Mar 2019 19:37:35 -0700 Subject: [PATCH 098/763] Allow using 'source -- file' (fixes #1518) --- src/ShellCheck/Checker.hs | 3 +++ src/ShellCheck/Parser.hs | 10 +++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Checker.hs b/src/ShellCheck/Checker.hs index 8b3eb76..471c364 100644 --- a/src/ShellCheck/Checker.hs +++ b/src/ShellCheck/Checker.hs @@ -204,6 +204,9 @@ prop_failsWhenNotSourcing = prop_worksWhenSourcing = null $ checkWithIncludes [("lib", "bar=1")] "source lib; echo \"$bar\"" +prop_worksWhenSourcingWithDashDash = + null $ checkWithIncludes [("lib", "bar=1")] "source -- lib; echo \"$bar\"" + prop_worksWhenDotting = null $ checkWithIncludes [("lib", "bar=1")] ". lib; echo \"$bar\"" diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 0289e0d..4ecf602 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -2062,7 +2062,8 @@ readSimpleCommand = called "simple command" $ do readSource :: Monad m => Token -> SCParser m Token -readSource t@(T_Redirecting _ _ (T_SimpleCommand cmdId _ (cmd:file:_))) = do +readSource t@(T_Redirecting _ _ (T_SimpleCommand cmdId _ (cmd:file':rest'))) = do + let file = getFile file' rest' override <- getSourceOverride let literalFile = do name <- override `mplus` getLiteralString file @@ -2107,6 +2108,13 @@ readSource t@(T_Redirecting _ _ (T_SimpleCommand cmdId _ (cmd:file:_))) = do included <|> failed where + getFile :: Token -> [Token] -> Token + getFile file (next:rest) = + case getLiteralString file of + Just "--" -> next + x -> file + getFile file _ = file + subRead name script = withContext (ContextSource name) $ inSeparateContext $ From f514f5f7351756a6c39d34c6700042133cb59541 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 20 Mar 2019 22:10:04 -0700 Subject: [PATCH 099/763] Warn about flipped $ and " in $"(cmd)" (fixes #1517) --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 586ede6..434ce79 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ - Preliminary support for fix suggestions - Files containing Bats tests can now be checked - Directory wide directives can now be placed in a `.shellcheckrc` +- SC2247: Warn about $"(cmd)" and $"{var}" - SC2246: Warn if a shebang's interpreter ends with / - SC2245: Warn that Ksh ignores all but the first glob result in `[` - SC2243/SC2244: Suggest using explicit -n for `[ $foo ]` diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 7498bb8..00842c1 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -171,6 +171,7 @@ nodeChecks = [ ,checkInvertedStringTest ,checkRedirectionToCommand ,checkNullaryExpansionTest + ,checkDollarQuoteParen ] @@ -3169,5 +3170,18 @@ checkNullaryExpansionTest params t = fix = fixWith [replaceStart id params 0 "-n "] _ -> return () + +prop_checkDollarQuoteParen1 = verify checkDollarQuoteParen "$\"(foo)\"" +prop_checkDollarQuoteParen2 = verify checkDollarQuoteParen "$\"{foo}\"" +prop_checkDollarQuoteParen3 = verifyNot checkDollarQuoteParen "\"$(foo)\"" +prop_checkDollarQuoteParen4 = verifyNot checkDollarQuoteParen "$\"..\"" +checkDollarQuoteParen params t = + case t of + T_DollarDoubleQuoted id ((T_Literal _ (c:_)):_) | c `elem` "({" -> + warnWithFix id 2247 "Flip leading $ and \" if this should be a quoted substitution." (fix id) + _ -> return () + where + fix id = fixWith [replaceStart id params 2 "\"$"] + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) From 9652ccfdbdbfee1290ace8137edc08f94bd25f77 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 13 Apr 2019 13:10:48 -0700 Subject: [PATCH 100/763] Add a verbose mode: `-S verbose` --- CHANGELOG.md | 1 + shellcheck.1.md | 5 +++-- shellcheck.hs | 5 +++-- src/ShellCheck/Formatter/Format.hs | 1 + src/ShellCheck/Formatter/TTY.hs | 1 + src/ShellCheck/Interface.hs | 4 ++-- 6 files changed, 11 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 434ce79..00faafc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ - Preliminary support for fix suggestions - Files containing Bats tests can now be checked - Directory wide directives can now be placed in a `.shellcheckrc` +- Verbose mode: Use `-S verbose` for especially pedantic suggestions - SC2247: Warn about $"(cmd)" and $"{var}" - SC2246: Warn if a shebang's interpreter ends with / - SC2245: Warn that Ksh ignores all but the first glob result in `[` diff --git a/shellcheck.1.md b/shellcheck.1.md index 1499c80..b533396 100644 --- a/shellcheck.1.md +++ b/shellcheck.1.md @@ -69,8 +69,9 @@ not warn at all, as `ksh` supports decimals in arithmetic contexts. **-S**\ *SEVERITY*,\ **--severity=***severity* -: Specify minimum severity of errors to consider. Valid values are *error*, - *warning*, *info* and *style*. The default is *style*. +: Specify minimum severity of errors to consider. Valid values in order of + severity are *error*, *warning*, *info*, *style* and *verbose*. + The default is *style*. **-s**\ *shell*,\ **--shell=***shell* diff --git a/shellcheck.hs b/shellcheck.hs index 483da1b..06516e5 100644 --- a/shellcheck.hs +++ b/shellcheck.hs @@ -103,7 +103,7 @@ options = [ "Specify dialect (sh, bash, dash, ksh)", Option "S" ["severity"] (ReqArg (Flag "severity") "SEVERITY") - "Minimum severity of errors to consider (error, warning, info, style)", + "Minimum severity of errors to consider (error, warning, info, style, verbose)", Option "V" ["version"] (NoArg $ Flag "version" "true") "Print version information", Option "W" ["wiki-link-count"] @@ -254,7 +254,8 @@ parseSeverityOption value = ("error", ErrorC), ("warning", WarningC), ("info", InfoC), - ("style", StyleC) + ("style", StyleC), + ("verbose", VerboseC) ] parseOption flag options = diff --git a/src/ShellCheck/Formatter/Format.hs b/src/ShellCheck/Formatter/Format.hs index 57b9d71..bb513cd 100644 --- a/src/ShellCheck/Formatter/Format.hs +++ b/src/ShellCheck/Formatter/Format.hs @@ -47,6 +47,7 @@ severityText pc = WarningC -> "warning" InfoC -> "info" StyleC -> "style" + VerboseC -> "verbose" -- Realign comments from a tabstop of 8 to 1 makeNonVirtual comments contents = diff --git a/src/ShellCheck/Formatter/TTY.hs b/src/ShellCheck/Formatter/TTY.hs index 4e9c272..c0d5841 100644 --- a/src/ShellCheck/Formatter/TTY.hs +++ b/src/ShellCheck/Formatter/TTY.hs @@ -57,6 +57,7 @@ colorForLevel level = "warning" -> 33 -- yellow "info" -> 32 -- green "style" -> 32 -- green + "verbose" -> 32 -- green "message" -> 1 -- bold "source" -> 0 -- none _ -> 0 -- none diff --git a/src/ShellCheck/Interface.hs b/src/ShellCheck/Interface.hs index 3a6d685..0661386 100644 --- a/src/ShellCheck/Interface.hs +++ b/src/ShellCheck/Interface.hs @@ -32,7 +32,7 @@ module ShellCheck.Interface , ExecutionMode(Executed, Sourced) , ErrorMessage , Code - , Severity(ErrorC, WarningC, InfoC, StyleC) + , Severity(ErrorC, WarningC, InfoC, StyleC, VerboseC) , Position(posFile, posLine, posColumn) , Comment(cSeverity, cCode, cMessage) , PositionedComment(pcStartPos , pcEndPos , pcComment, pcFix) @@ -189,7 +189,7 @@ data ExecutionMode = Executed | Sourced deriving (Show, Eq) type ErrorMessage = String type Code = Integer -data Severity = ErrorC | WarningC | InfoC | StyleC +data Severity = ErrorC | WarningC | InfoC | StyleC | VerboseC deriving (Show, Eq, Ord, Generic, NFData) data Position = Position { posFile :: String, -- Filename From c860b74505bdc38850384bd56bf2fdb7021a33ac Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 13 Apr 2019 13:40:18 -0700 Subject: [PATCH 101/763] Set SC2243/SC2244 level to "verbose" --- CHANGELOG.md | 2 +- src/ShellCheck/Analytics.hs | 4 ++-- src/ShellCheck/AnalyzerLib.hs | 2 ++ 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 00faafc..71db82c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ - SC2247: Warn about $"(cmd)" and $"{var}" - SC2246: Warn if a shebang's interpreter ends with / - SC2245: Warn that Ksh ignores all but the first glob result in `[` -- SC2243/SC2244: Suggest using explicit -n for `[ $foo ]` +- SC2243/SC2244: Suggest using explicit -n for `[ $foo ]` (verbose) ### Changed - If a directive or shebang is not specified, a `.bash/.bats/.dash/.ksh` diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 00842c1..bd7146a 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -3159,11 +3159,11 @@ checkNullaryExpansionTest params t = TC_Nullary _ _ word -> case getWordParts word of [t] | isCommandSubstitution t -> - styleWithFix id 2243 "Prefer explicit -n to check for output (or run command without [/[[ to check for success)." fix + verboseWithFix id 2243 "Prefer explicit -n to check for output (or run command without [/[[ to check for success)." fix -- If they're constant, you get SC2157 &co x | all (not . isConstant) x -> - styleWithFix id 2244 "Prefer explicit -n to check non-empty string (or use =/-ne to check boolean/integer)." fix + verboseWithFix id 2244 "Prefer explicit -n to check non-empty string (or use =/-ne to check boolean/integer)." fix _ -> return () where id = getId word diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 01c87e4..e52f4e3 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -159,6 +159,8 @@ warnWithFix :: MonadWriter [TokenComment] m => Id -> Code -> String -> Fix -> m warnWithFix = addCommentWithFix WarningC styleWithFix :: MonadWriter [TokenComment] m => Id -> Code -> String -> Fix -> m () styleWithFix = addCommentWithFix StyleC +verboseWithFix :: MonadWriter [TokenComment] m => Id -> Code -> String -> Fix -> m () +verboseWithFix = addCommentWithFix VerboseC addCommentWithFix :: MonadWriter [TokenComment] m => Severity -> Id -> Code -> String -> Fix -> m () addCommentWithFix severity id code str fix = From b76c0a822124e802e35750a248a6f5da43d536f6 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 13 Apr 2019 20:19:13 -0700 Subject: [PATCH 102/763] SC2248: Warn about unquoted variables without special chars --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 45 ++++++++++++++++++++++++++----------- src/ShellCheck/Data.hs | 6 +++-- 3 files changed, 37 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 71db82c..558d239 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - Files containing Bats tests can now be checked - Directory wide directives can now be placed in a `.shellcheckrc` - Verbose mode: Use `-S verbose` for especially pedantic suggestions +- SC2248: Warn about unquoted variables without special chars (verbose) - SC2247: Warn about $"(cmd)" and $"{var}" - SC2246: Warn if a shebang's interpreter ends with / - SC2245: Warn that Ksh ignores all but the first glob result in `[` diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index bd7146a..c6f61a4 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -53,7 +53,7 @@ treeChecks = [ (\p t -> (mapM_ ((\ f -> f t) . (\ f -> f p)) nodeChecks)) ,subshellAssignmentCheck - ,checkSpacefulness + ,checkVerboseSpacefulness ,checkQuotesInLiterals ,checkShebangParameters ,checkFunctionsUsedExternally @@ -1639,10 +1639,12 @@ prop_checkSpacefulness2 = verifyNotTree checkSpacefulness "a='cow moo'; [[ $a ]] prop_checkSpacefulness3 = verifyNotTree checkSpacefulness "a='cow*.mp3'; echo \"$a\"" prop_checkSpacefulness4 = verifyTree checkSpacefulness "for f in *.mp3; do echo $f; done" prop_checkSpacefulness4a= verifyNotTree checkSpacefulness "foo=3; foo=$(echo $foo)" +prop_checkSpacefulness4v= verifyTree checkVerboseSpacefulness "foo=3; foo=$(echo $foo)" prop_checkSpacefulness5 = verifyTree checkSpacefulness "a='*'; b=$a; c=lol${b//foo/bar}; echo $c" prop_checkSpacefulness6 = verifyTree checkSpacefulness "a=foo$(lol); echo $a" prop_checkSpacefulness7 = verifyTree checkSpacefulness "a=foo\\ bar; rm $a" prop_checkSpacefulness8 = verifyNotTree checkSpacefulness "a=foo\\ bar; a=foo; rm $a" +prop_checkSpacefulness8v= verifyTree checkVerboseSpacefulness "a=foo\\ bar; a=foo; rm $a" prop_checkSpacefulness10= verifyTree checkSpacefulness "rm $1" prop_checkSpacefulness11= verifyTree checkSpacefulness "rm ${10//foo/bar}" prop_checkSpacefulness12= verifyNotTree checkSpacefulness "(( $1 + 3 ))" @@ -1662,6 +1664,7 @@ prop_checkSpacefulness25= verifyTree checkSpacefulness "a='s/[0-9]//g'; sed $a" prop_checkSpacefulness26= verifyTree checkSpacefulness "a='foo bar'; echo {1,2,$a}" prop_checkSpacefulness27= verifyNotTree checkSpacefulness "echo ${a:+'foo'}" prop_checkSpacefulness28= verifyNotTree checkSpacefulness "exec {n}>&1; echo $n" +prop_checkSpacefulness28v = verifyTree checkVerboseSpacefulness "exec {n}>&1; echo $n" prop_checkSpacefulness29= verifyNotTree checkSpacefulness "n=$(stuff); exec {n}>&-;" prop_checkSpacefulness30= verifyTree checkSpacefulness "file='foo bar'; echo foo > $file;" prop_checkSpacefulness31= verifyNotTree checkSpacefulness "echo \"`echo \\\"$1\\\"`\"" @@ -1670,9 +1673,15 @@ prop_checkSpacefulness33= verifyTree checkSpacefulness "for file; do echo $file; prop_checkSpacefulness34= verifyTree checkSpacefulness "declare foo$n=$1" prop_checkSpacefulness35= verifyNotTree checkSpacefulness "echo ${1+\"$1\"}" prop_checkSpacefulness36= verifyNotTree checkSpacefulness "arg=$#; echo $arg" +prop_checkSpacefulness36v = verifyTree checkVerboseSpacefulness "arg=$#; echo $arg" prop_checkSpacefulness37= verifyNotTree checkSpacefulness "@test 'status' {\n [ $status -eq 0 ]\n}" +prop_checkSpacefulness37v = verifyTree checkVerboseSpacefulness "@test 'status' {\n [ $status -eq 0 ]\n}" -checkSpacefulness params t = +-- This is slightly awkward because we want the tests to +-- discriminate between normal and verbose output. +checkSpacefulness params t = checkSpacefulness' False params t +checkVerboseSpacefulness params t = checkSpacefulness' True params t +checkSpacefulness' alsoVerbose params t = doVariableFlowAnalysis readF writeF (Map.fromList defaults) (variableFlow params) where defaults = zip variablesWithoutSpaces (repeat False) @@ -1686,23 +1695,33 @@ checkSpacefulness params t = readF _ token name = do spaces <- hasSpaces name - return [warning | - isExpansion token && spaces + let needsQuoting = + isExpansion token && not (isArrayExpansion token) -- There's another warning for this && not (isCountingReference token) && not (isQuoteFree parents token) && not (isQuotedAlternativeReference token) - && not (usedAsCommandName parents token)] - where - warning = - if isDefaultAssignment (parentMap params) token + && not (usedAsCommandName parents token) + + return . execWriter $ when needsQuoting $ + if spaces then - makeComment InfoC (getId token) 2223 - "This default assignment may cause DoS due to globbing. Quote it." + if isDefaultAssignment (parentMap params) token + then + emit $ makeComment InfoC (getId token) 2223 + "This default assignment may cause DoS due to globbing. Quote it." + else + emit $ makeCommentWithFix InfoC (getId token) 2086 + "Double quote to prevent globbing and word splitting." + (fixFor token) else - makeCommentWithFix InfoC (getId token) 2086 - "Double quote to prevent globbing and word splitting." - (surroundWidth (getId token) params "\"") + when (alsoVerbose && name `notElem` specialVariablesWithoutSpaces) $ + emit $ makeCommentWithFix VerboseC (getId token) 2248 + "Prefer double quoting even when variables don't contain special characters." + (fixFor token) + where + fixFor token = (surroundWidth (getId token) params "\"") + emit x = tell [x] writeF _ _ name (DataString SourceExternal) = setSpaces name True >> return [] writeF _ _ name (DataString SourceInteger) = setSpaces name False >> return [] diff --git a/src/ShellCheck/Data.hs b/src/ShellCheck/Data.hs index e4ef675..cae07d3 100644 --- a/src/ShellCheck/Data.hs +++ b/src/ShellCheck/Data.hs @@ -38,8 +38,10 @@ internalVariables = [ , ".sh.version" ] -variablesWithoutSpaces = [ - "$", "-", "?", "!", "#", +specialVariablesWithoutSpaces = [ + "$", "-", "?", "!", "#" + ] +variablesWithoutSpaces = specialVariablesWithoutSpaces ++ [ "BASHPID", "BASH_ARGC", "BASH_LINENO", "BASH_SUBSHELL", "EUID", "LINENO", "OPTIND", "PPID", "RANDOM", "SECONDS", "SHELLOPTS", "SHLVL", "UID", "COLUMNS", "HISTFILESIZE", "HISTSIZE", "LINES" From 5b7354918fef77792c7b48b81911cccf2af4c070 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 14 Apr 2019 16:19:33 -0700 Subject: [PATCH 103/763] SC2249: When verbose, warn about missing default case (fixes #997) --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 18 ++++++++++++++++++ src/ShellCheck/AnalyzerLib.hs | 1 + 3 files changed, 20 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 558d239..0dc5bb8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - Files containing Bats tests can now be checked - Directory wide directives can now be placed in a `.shellcheckrc` - Verbose mode: Use `-S verbose` for especially pedantic suggestions +- SC2249: Warn about `case` with missing default case (verbose) - SC2248: Warn about unquoted variables without special chars (verbose) - SC2247: Warn about $"(cmd)" and $"{var}" - SC2246: Warn if a shebang's interpreter ends with / diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index c6f61a4..fb7c0e0 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -172,6 +172,7 @@ nodeChecks = [ ,checkRedirectionToCommand ,checkNullaryExpansionTest ,checkDollarQuoteParen + ,checkDefaultCase ] @@ -3202,5 +3203,22 @@ checkDollarQuoteParen params t = where fix id = fixWith [replaceStart id params 2 "\"$"] +prop_checkDefaultCase1 = verify checkDefaultCase "case $1 in a) true ;; esac" +prop_checkDefaultCase2 = verify checkDefaultCase "case $1 in ?*?) true ;; *? ) true ;; esac" +prop_checkDefaultCase3 = verifyNot checkDefaultCase "case $1 in x|*) true ;; esac" +prop_checkDefaultCase4 = verifyNot checkDefaultCase "case $1 in **) true ;; esac" +checkDefaultCase _ t = + case t of + T_CaseExpression id _ list -> + unless (any canMatchAny list) $ + verbose id 2249 "Consider adding a default *) case, even if it just exits with error." + _ -> return () + where + canMatchAny (_, list, _) = any canMatchAny' list + -- hlint objects to 'pattern' as a variable name + canMatchAny' pat = fromMaybe False $ do + pg <- wordToExactPseudoGlob pat + return $ pseudoGlobIsSuperSetof pg [PGMany] + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index e52f4e3..c4f7cfa 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -154,6 +154,7 @@ warn id code str = addComment $ makeComment WarningC id code str err id code str = addComment $ makeComment ErrorC id code str info id code str = addComment $ makeComment InfoC id code str style id code str = addComment $ makeComment StyleC id code str +verbose id code str = addComment $ makeComment VerboseC id code str warnWithFix :: MonadWriter [TokenComment] m => Id -> Code -> String -> Fix -> m () warnWithFix = addCommentWithFix WarningC From b82429496186220fc14a0ae3b83221fad4105b87 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 14 Apr 2019 20:58:01 -0700 Subject: [PATCH 104/763] Limit SC2032 to likely command args (fixes #1537) --- src/ShellCheck/Analytics.hs | 53 ++++++++++++++++++++++++++----------- 1 file changed, 38 insertions(+), 15 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index fb7c0e0..3ef42d2 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1833,29 +1833,51 @@ checkQuotesInLiterals params t = prop_checkFunctionsUsedExternally1 = verifyTree checkFunctionsUsedExternally "foo() { :; }; sudo foo" prop_checkFunctionsUsedExternally2 = - verifyTree checkFunctionsUsedExternally "alias f='a'; xargs -n 1 f" + verifyTree checkFunctionsUsedExternally "alias f='a'; xargs -0 f" +prop_checkFunctionsUsedExternally2b= + verifyNotTree checkFunctionsUsedExternally "alias f='a'; find . -type f" +prop_checkFunctionsUsedExternally2c= + verifyTree checkFunctionsUsedExternally "alias f='a'; find . -type f -exec f +" prop_checkFunctionsUsedExternally3 = verifyNotTree checkFunctionsUsedExternally "f() { :; }; echo f" prop_checkFunctionsUsedExternally4 = verifyNotTree checkFunctionsUsedExternally "foo() { :; }; sudo \"foo\"" +prop_checkFunctionsUsedExternally5 = + verifyTree checkFunctionsUsedExternally "foo() { :; }; ssh host foo" +prop_checkFunctionsUsedExternally6 = + verifyNotTree checkFunctionsUsedExternally "foo() { :; }; ssh host echo foo" +prop_checkFunctionsUsedExternally7 = + verifyNotTree checkFunctionsUsedExternally "install() { :; }; sudo apt-get install foo" checkFunctionsUsedExternally params t = runNodeAnalysis checkCommand params t where - invokingCmds = [ - "chroot", - "find", - "screen", - "ssh", - "su", - "sudo", - "xargs" - ] checkCommand _ t@(T_SimpleCommand _ _ (cmd:args)) = - let name = fromMaybe "" $ getCommandBasename t in - when (name `elem` invokingCmds) $ - mapM_ (checkArg name) args + case getCommandBasename t of + Just name -> do + let argStrings = map (\x -> (fromMaybe "" $ getLiteralString x, x)) args + let candidates = getPotentialCommands name argStrings + mapM_ (checkArg name) candidates + _ -> return () checkCommand _ _ = return () + -- Try to pick out the argument[s] that may be commands + getPotentialCommands name argAndString = + case name of + "chroot" -> firstNonFlag + "screen" -> firstNonFlag + "sudo" -> firstNonFlag + "xargs" -> firstNonFlag + "tmux" -> firstNonFlag + "ssh" -> take 1 $ drop 1 $ dropFlags argAndString + "find" -> take 1 $ drop 1 $ + dropWhile (\x -> fst x `notElem` findExecFlags) argAndString + _ -> [] + where + firstNonFlag = take 1 $ dropFlags argAndString + findExecFlags = ["-exec", "-execdir", "-ok"] + dropFlags = dropWhile (\x -> "-" `isPrefixOf` fst x) + + -- Make a map from functions/aliases to definition IDs analyse f t = execState (doAnalysis f t) [] functions = Map.fromList $ analyse findFunctions t findFunctions (T_Function id _ _ name _) = modify ((name, id):) @@ -1863,10 +1885,11 @@ checkFunctionsUsedExternally params t = | t `isUnqualifiedCommand` "alias" = mapM_ getAlias args findFunctions _ = return () getAlias arg = - let string = concat $ oversimplify arg + let string = onlyLiteralString arg in when ('=' `elem` string) $ modify ((takeWhile (/= '=') string, getId arg):) - checkArg cmd arg = potentially $ do + + checkArg cmd (_, arg) = potentially $ do literalArg <- getUnquotedLiteral arg -- only consider unquoted literals definitionId <- Map.lookup literalArg functions return $ do From cef4c1a0bc6889e383cebecf757c854325ff2b1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9E=97=E5=8D=9A=E4=BB=81=28Buo-ren=20Lin=29?= Date: Tue, 16 Apr 2019 17:48:34 +0800 Subject: [PATCH 105/763] snap: Flip `grade` property to stable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This allows the snap to be promoted to the `candidate` and the `stable` release channels Signed-off-by: 林博仁(Buo-ren Lin) --- snap/snapcraft.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 9c50293..a3c41d1 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -23,7 +23,7 @@ description: | # snap connect shellcheck:removable-media version: git -grade: devel +grade: stable confinement: strict apps: From 67dbbcbd89bb5f58eaf0e01605fcc3e299361730 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9E=97=E5=8D=9A=E4=BB=81=28Buo-ren=20Lin=29?= Date: Tue, 16 Apr 2019 17:53:01 +0800 Subject: [PATCH 106/763] snap: Drop unneeded trailing slash in the `source` property MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 林博仁(Buo-ren Lin) --- snap/snapcraft.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index a3c41d1..83ae66d 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -34,7 +34,7 @@ apps: parts: shellcheck: plugin: dump - source: ./ + source: . build-packages: - cabal-install - squid3 From 10955a143c81dae6cf7f82cd6dc21a3b9479b30b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9E=97=E5=8D=9A=E4=BB=81=28Buo-ren=20Lin=29?= Date: Tue, 16 Apr 2019 18:10:49 +0800 Subject: [PATCH 107/763] snap: Replace deprecated build and install keyword MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These keywords has been obsoleted in Snapcraft 3. Refer-to: The 'build' keyword has been replaced by 'override-build' Refer-to: The 'install' keyword has been replaced by 'override-build' Signed-off-by: 林博仁(Buo-ren Lin) --- snap/snapcraft.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 83ae66d..73d0eeb 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -38,7 +38,7 @@ parts: build-packages: - cabal-install - squid3 - build: | + override-build: | # See comments in .snapsquid.conf [ "$http_proxy" ] && { squid3 -f .snapsquid.conf @@ -48,6 +48,6 @@ parts: cabal sandbox init cabal update || cat /var/log/squid/* cabal install -j - install: | + install -d $SNAPCRAFT_PART_INSTALL/usr/bin install .cabal-sandbox/bin/shellcheck $SNAPCRAFT_PART_INSTALL/usr/bin From 025c380b8410459f72d806420d408d8f4f9cd956 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9E=97=E5=8D=9A=E4=BB=81=28Buo-ren=20Lin=29?= Date: Tue, 16 Apr 2019 19:59:29 +0800 Subject: [PATCH 108/763] snap: Migrate to core18 base MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch migrates the snap to core18 base, which should make the cabal build work again. Signed-off-by: 林博仁(Buo-ren Lin) --- snap/snapcraft.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 73d0eeb..8365b13 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -23,6 +23,7 @@ description: | # snap connect shellcheck:removable-media version: git +base: core18 grade: stable confinement: strict @@ -37,7 +38,7 @@ parts: source: . build-packages: - cabal-install - - squid3 + - squid override-build: | # See comments in .snapsquid.conf [ "$http_proxy" ] && { From af46758ff17dab17d23a1f30b92a0c470bc53f29 Mon Sep 17 00:00:00 2001 From: Pontus Andersson Date: Mon, 22 Apr 2019 14:34:38 +0200 Subject: [PATCH 109/763] Add option to look for sources in alternate root paths Add a new optional flag "-r|--root ROOTPATHS", where ROOTPATHS is a colon separated list of paths, that will look for external sources in alternate roots. This is particular useful when the run-time environment does not fully match the development environment. The #shellcheck source=file directive is useful, but has its limitations in certain scenarios. Also, in many cases the directive could be removed from scripts when the root flag is used. Script example.bash: #!/bin/bash source /etc/foo/config Example usage where etc/foo/config exists in skel/foo: # shellcheck -x -r skel/foo:skel/core example.bash --- shellcheck.hs | 30 ++++++++++++++++++++++++++++++ src/ShellCheck/Interface.hs | 4 ++++ src/ShellCheck/Parser.hs | 4 +++- 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/shellcheck.hs b/shellcheck.hs index 06516e5..c57c379 100644 --- a/shellcheck.hs +++ b/shellcheck.hs @@ -69,6 +69,7 @@ instance Monoid Status where data Options = Options { checkSpec :: CheckSpec, externalSources :: Bool, + rootPaths :: [FilePath], formatterOptions :: FormatterOptions, minSeverity :: Severity } @@ -76,6 +77,7 @@ data Options = Options { defaultOptions = Options { checkSpec = emptyCheckSpec, externalSources = False, + rootPaths = [], formatterOptions = newFormatterOptions { foColorOption = ColorAuto }, @@ -98,6 +100,9 @@ options = [ "Output format (" ++ formatList ++ ")", Option "" ["norc"] (NoArg $ Flag "norc" "true") "Don't look for .shellcheckrc files", + Option "r" ["root"] + (ReqArg (Flag "root") "ROOTPATHS") + "Specify alternate root path(s) when looking for sources (colon separated)", Option "s" ["shell"] (ReqArg (Flag "shell") "SHELLNAME") "Specify dialect (sh, bash, dash, ksh)", @@ -311,6 +316,12 @@ parseOption flag options = } } + Flag "root" str -> do + let paths = filter (not . null) $ split ':' str + return options { + rootPaths = paths + } + Flag "sourced" _ -> return options { checkSpec = (checkSpec options) { @@ -362,8 +373,10 @@ ioInterface options files = do inputs <- mapM normalize files cache <- newIORef emptyCache configCache <- newIORef ("", Nothing) + let rootPathsCache = rootPaths options return SystemInterface { siReadFile = get cache inputs, + siFindSource = findSourceFile rootPathsCache, siGetConfig = getConfig configCache } where @@ -455,6 +468,23 @@ ioInterface options files = do putStrLn $ file ++ ": " ++ show err return ("", True) + findSourceFile rootPaths file = do + case file of + ('/':root) -> do + source <- find root + return source + _ -> + return file + where + find root = do + sources <- filterM doesFileExist paths + case sources of + [] -> return file + (first:_) -> return first + where + paths = map join rootPaths + join path = joinPath [path, root] + inputFile file = do (handle, shouldCache) <- if file == "-" diff --git a/src/ShellCheck/Interface.hs b/src/ShellCheck/Interface.hs index 0661386..8b0ba0f 100644 --- a/src/ShellCheck/Interface.hs +++ b/src/ShellCheck/Interface.hs @@ -73,6 +73,8 @@ import qualified Data.Map as Map data SystemInterface m = SystemInterface { -- Read a file by filename, or return an error siReadFile :: String -> m (Either ErrorMessage String), + -- Find source file in alternate root paths + siFindSource :: String -> m (FilePath), -- Get the configuration file (name, contents) for a filename siGetConfig :: String -> m (Maybe (FilePath, String)) } @@ -287,6 +289,7 @@ data ColorOption = mockedSystemInterface :: [(String, String)] -> SystemInterface Identity mockedSystemInterface files = SystemInterface { siReadFile = rf, + siFindSource = fs, siGetConfig = const $ return Nothing } where @@ -294,6 +297,7 @@ mockedSystemInterface files = SystemInterface { case filter ((== file) . fst) files of [] -> return $ Left "File not included in mock." [(_, contents)] -> return $ Right contents + fs file = return file mockRcFile rcfile mock = mock { siGetConfig = const . return $ Just (".shellcheckrc", rcfile) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 4ecf602..2abdb36 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -2087,7 +2087,9 @@ readSource t@(T_Redirecting _ _ (T_SimpleCommand cmdId _ (cmd:file':rest'))) = d input <- if filename == "/dev/null" -- always allow /dev/null then return (Right "") - else system $ siReadFile sys filename + else do + filename' <- system $ siFindSource sys filename + system $ siReadFile sys filename' case input of Left err -> do parseNoteAtId (getId file) InfoC 1091 $ From c6c12f52bdda4ee95db0b8bf65063a1d999ab65c Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 24 Apr 2019 18:51:24 -0700 Subject: [PATCH 110/763] Expand root paths into source paths --- CHANGELOG.md | 1 + shellcheck.1.md | 8 +++++ shellcheck.hs | 58 ++++++++++++++++++++----------------- src/ShellCheck/Checker.hs | 24 +++++++++++++++ src/ShellCheck/Interface.hs | 6 ++-- src/ShellCheck/Parser.hs | 3 +- 6 files changed, 70 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0dc5bb8..d81b547 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - Files containing Bats tests can now be checked - Directory wide directives can now be placed in a `.shellcheckrc` - Verbose mode: Use `-S verbose` for especially pedantic suggestions +- Source paths: Use `-P dir1:dir2` to specify path for sourced files - SC2249: Warn about `case` with missing default case (verbose) - SC2248: Warn about unquoted variables without special chars (verbose) - SC2247: Warn about $"(cmd)" and $"{var}" diff --git a/shellcheck.1.md b/shellcheck.1.md index b533396..92235f9 100644 --- a/shellcheck.1.md +++ b/shellcheck.1.md @@ -67,6 +67,14 @@ not warn at all, as `ksh` supports decimals in arithmetic contexts. : Don't try to look for .shellcheckrc configuration files. +**-P**\ *SOURCEPATH*,\ **--source-path=***SOURCEPATH* + +: Specify paths to search for sourced files, separated by `:` on Unix and + `;` on Windows. Absolute paths will also be rooted in these. The special + path `SCRIPTDIR` can be used to specify the currently checked script's + directory, as in `-P SCRIPTDIR` or `-P SCRIPTDIR/../libs`. Subsequent + `-P` flags accumulate and take predecence. + **-S**\ *SEVERITY*,\ **--severity=***severity* : Specify minimum severity of errors to consider. Valid values in order of diff --git a/shellcheck.hs b/shellcheck.hs index c57c379..ac6639a 100644 --- a/shellcheck.hs +++ b/shellcheck.hs @@ -69,7 +69,7 @@ instance Monoid Status where data Options = Options { checkSpec :: CheckSpec, externalSources :: Bool, - rootPaths :: [FilePath], + sourcePaths :: [FilePath], formatterOptions :: FormatterOptions, minSeverity :: Severity } @@ -77,7 +77,7 @@ data Options = Options { defaultOptions = Options { checkSpec = emptyCheckSpec, externalSources = False, - rootPaths = [], + sourcePaths = [], formatterOptions = newFormatterOptions { foColorOption = ColorAuto }, @@ -100,9 +100,9 @@ options = [ "Output format (" ++ formatList ++ ")", Option "" ["norc"] (NoArg $ Flag "norc" "true") "Don't look for .shellcheckrc files", - Option "r" ["root"] - (ReqArg (Flag "root") "ROOTPATHS") - "Specify alternate root path(s) when looking for sources (colon separated)", + Option "P" ["source-path"] + (ReqArg (Flag "source-path") "SOURCEPATHS") + "Specify path when looking for sourced files (\"SCRIPTDIR\" for script's dir)", Option "s" ["shell"] (ReqArg (Flag "shell") "SHELLNAME") "Specify dialect (sh, bash, dash, ksh)", @@ -316,10 +316,10 @@ parseOption flag options = } } - Flag "root" str -> do - let paths = filter (not . null) $ split ':' str + Flag "source-path" str -> do + let paths = splitSearchPath str return options { - rootPaths = paths + sourcePaths = (sourcePaths options) ++ paths } Flag "sourced" _ -> @@ -373,10 +373,9 @@ ioInterface options files = do inputs <- mapM normalize files cache <- newIORef emptyCache configCache <- newIORef ("", Nothing) - let rootPathsCache = rootPaths options return SystemInterface { siReadFile = get cache inputs, - siFindSource = findSourceFile rootPathsCache, + siFindSource = findSourceFile inputs (sourcePaths options), siGetConfig = getConfig configCache } where @@ -468,22 +467,29 @@ ioInterface options files = do putStrLn $ file ++ ": " ++ show err return ("", True) - findSourceFile rootPaths file = do - case file of - ('/':root) -> do - source <- find root - return source - _ -> - return file - where - find root = do - sources <- filterM doesFileExist paths - case sources of - [] -> return file - (first:_) -> return first - where - paths = map join rootPaths - join path = joinPath [path, root] + andM a b arg = do + first <- a arg + if not first then return False else b arg + + findSourceFile inputs sourcePaths currentScript original = + if isAbsolute original + then + let (_, relative) = splitDrive original + in find relative original + else + find original original + where + find filename deflt = do + sources <- filterM ((allowable inputs) `andM` doesFileExist) + (map ( filename) $ map adjustPath sourcePaths) + case sources of + [] -> return deflt + (first:_) -> return first + scriptdir = dropFileName currentScript + adjustPath str = + case (splitDirectories str) of + ("SCRIPTDIR":rest) -> joinPath (scriptdir:rest) + _ -> str inputFile file = do (handle, shouldCache) <- diff --git a/src/ShellCheck/Checker.hs b/src/ShellCheck/Checker.hs index 471c364..b6ee068 100644 --- a/src/ShellCheck/Checker.hs +++ b/src/ShellCheck/Checker.hs @@ -150,6 +150,11 @@ checkOptionIncludes includes src = checkWithRc rc = getErrors (mockRcFile rc $ mockedSystemInterface []) +checkWithIncludesAndSourcePath includes mapper = getErrors + (mockedSystemInterface includes) { + siFindSource = mapper + } + prop_findsParseIssue = check "echo \"$12\"" == [1037] prop_commentDisablesParseIssue1 = @@ -335,5 +340,24 @@ prop_brokenRcGetsWarning = result == [1134, 2086] csIgnoreRC = False } +prop_sourcePathRedirectsName = result == [2086] + where + f "dir/myscript" "lib" = return "foo/lib" + result = checkWithIncludesAndSourcePath [("foo/lib", "echo $1")] f emptyCheckSpec { + csScript = "#!/bin/bash\nsource lib", + csFilename = "dir/myscript", + csCheckSourced = True + } + +prop_sourcePathRedirectsDirective = result == [2086] + where + f "dir/myscript" "lib" = return "foo/lib" + f _ _ = return "/dev/null" + result = checkWithIncludesAndSourcePath [("foo/lib", "echo $1")] f emptyCheckSpec { + csScript = "#!/bin/bash\n# shellcheck source=lib\nsource kittens", + csFilename = "dir/myscript", + csCheckSourced = True + } + return [] runTests = $quickCheckAll diff --git a/src/ShellCheck/Interface.hs b/src/ShellCheck/Interface.hs index 8b0ba0f..fa342e2 100644 --- a/src/ShellCheck/Interface.hs +++ b/src/ShellCheck/Interface.hs @@ -73,8 +73,8 @@ import qualified Data.Map as Map data SystemInterface m = SystemInterface { -- Read a file by filename, or return an error siReadFile :: String -> m (Either ErrorMessage String), - -- Find source file in alternate root paths - siFindSource :: String -> m (FilePath), + -- Given the current script and a sourced file, find the sourced file + siFindSource :: String -> String -> m FilePath, -- Get the configuration file (name, contents) for a filename siGetConfig :: String -> m (Maybe (FilePath, String)) } @@ -297,7 +297,7 @@ mockedSystemInterface files = SystemInterface { case filter ((== file) . fst) files of [] -> return $ Left "File not included in mock." [(_, contents)] -> return $ Right contents - fs file = return file + fs _ file = return file mockRcFile rcfile mock = mock { siGetConfig = const . return $ Just (".shellcheckrc", rcfile) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 2abdb36..75e45ce 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -2088,7 +2088,8 @@ readSource t@(T_Redirecting _ _ (T_SimpleCommand cmdId _ (cmd:file':rest'))) = d if filename == "/dev/null" -- always allow /dev/null then return (Right "") else do - filename' <- system $ siFindSource sys filename + currentScript <- Mr.asks currentFilename + filename' <- system $ siFindSource sys currentScript filename system $ siReadFile sys filename' case input of Left err -> do From bf1003eae36589d7cc78cb1cca210257620e0d60 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 27 Apr 2019 15:20:07 -0700 Subject: [PATCH 111/763] Auto-disable SC2119 when disabling SC2120 (fixes #703) --- CHANGELOG.md | 1 + src/ShellCheck/ASTLib.hs | 10 ++++++++++ src/ShellCheck/Analytics.hs | 18 +++++++++++++----- src/ShellCheck/AnalyzerLib.hs | 11 +++++------ 4 files changed, 29 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d81b547..79f7910 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ ### Changed - If a directive or shebang is not specified, a `.bash/.bats/.dash/.ksh` extension will be used to infer the shell type when present. +- Disabling SC2120 on a function now disables SC2119 on call sites ## v0.6.0 - 2018-12-02 ### Added diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 66269bc..7e8023e 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -501,3 +501,13 @@ isCommandSubstitution t = case t of T_DollarBraceCommandExpansion {} -> True T_Backticked {} -> True _ -> False + + +-- Is this a T_Annotation that ignores a specific code? +isAnnotationIgnoringCode code t = + case t of + T_Annotation _ anns _ -> any hasNum anns + _ -> False + where + hasNum (DisableComment ts) = code == ts + hasNum _ = False diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 3ef42d2..163a3cc 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2304,6 +2304,7 @@ prop_checkUnpassedInFunctions9 = verifyNotTree checkUnpassedInFunctions "foo() { prop_checkUnpassedInFunctions10= verifyNotTree checkUnpassedInFunctions "foo() { echo $!; }; foo;" prop_checkUnpassedInFunctions11= verifyNotTree checkUnpassedInFunctions "foo() { bar() { echo $1; }; bar baz; }; foo;" prop_checkUnpassedInFunctions12= verifyNotTree checkUnpassedInFunctions "foo() { echo ${!var*}; }; foo;" +prop_checkUnpassedInFunctions13= verifyNotTree checkUnpassedInFunctions "# shellcheck disable=SC2120\nfoo() { echo $1; }\nfoo\n" checkUnpassedInFunctions params root = execWriter $ mapM_ warnForGroup referenceGroups where @@ -2355,17 +2356,24 @@ checkUnpassedInFunctions params root = updateWith x@(name, _, _) = Map.insertWith (++) name [x] warnForGroup group = - when (all isArgumentless group) $ do - mapM_ suggestParams group - warnForDeclaration group + when (all isArgumentless group) $ + -- Allow ignoring SC2120 on the function to ignore all calls + let (name, func) = getFunction group + ignoring = shouldIgnoreCode params 2120 func + in unless ignoring $ do + mapM_ suggestParams group + warnForDeclaration func name suggestParams (name, _, thing) = info (getId thing) 2119 $ "Use " ++ name ++ " \"$@\" if function's $1 should mean script's $1." - warnForDeclaration ((name, _, _):_) = - warn (getId . fromJust $ Map.lookup name functionMap) 2120 $ + warnForDeclaration func name = + warn (getId func) 2120 $ name ++ " references arguments, but none are ever passed." + getFunction ((name, _, _):_) = + (name, fromJust $ Map.lookup name functionMap) + prop_checkOverridingPath1 = verify checkOverridingPath "PATH=\"$var/$foo\"" prop_checkOverridingPath2 = verify checkOverridingPath "PATH=\"mydir\"" diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index c4f7cfa..361ce8b 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -882,16 +882,15 @@ filterByAnnotation asSpec params = shouldIgnore note = any (shouldIgnoreFor (getCode note)) $ getPath parents (T_Bang $ tcId note) - shouldIgnoreFor num (T_Annotation _ anns _) = - any hasNum anns - where - hasNum (DisableComment ts) = num == ts - hasNum _ = False shouldIgnoreFor _ T_Include {} = not $ asCheckSourced asSpec - shouldIgnoreFor _ _ = False + shouldIgnoreFor code t = isAnnotationIgnoringCode code t parents = parentMap params getCode = cCode . tcComment +shouldIgnoreCode params code t = + any (isAnnotationIgnoringCode code) $ + getPath (parentMap params) t + -- Is this a ${#anything}, to get string length or array count? isCountingReference (T_DollarBraced id token) = case concat $ oversimplify token of From 9470b9dc315c4b868aa174624ec649f4e1bcd478 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 27 Apr 2019 16:22:01 -0700 Subject: [PATCH 112/763] Don't mention arrays in SC2089 in sh/dash (fixes #1014) --- src/ShellCheck/Analytics.hs | 8 ++++++-- src/ShellCheck/AnalyzerLib.hs | 2 ++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 163a3cc..f89df9a 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1822,12 +1822,16 @@ checkQuotesInLiterals params t = && not (isQuoteFree parents expr) && not (squashesQuotes expr) then [ - makeComment WarningC (fromJust assignment) 2089 - "Quotes/backslashes will be treated literally. Use an array.", + makeComment WarningC (fromJust assignment) 2089 $ + "Quotes/backslashes will be treated literally. " ++ suggestion, makeComment WarningC (getId expr) 2090 "Quotes/backslashes in this variable will not be respected." ] else []) + suggestion = + if supportsArrays (shellType params) + then "Use an array." + else "Rewrite using set/\"$@\" or functions." prop_checkFunctionsUsedExternally1 = diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 361ce8b..01fcc8f 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -940,5 +940,7 @@ getOpts flagTokenizer string cmd = process flags more <- process rest2 return $ (flag1, token1) : more +supportsArrays shell = shell == Bash || shell == Ksh + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) From 6ccf9d6af1e3bfa1189ea8a1f8d1fd3515e25b7c Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 27 Apr 2019 17:25:20 -0700 Subject: [PATCH 113/763] Mention in manual that 'sh' means POSIX and not system --- shellcheck.1.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/shellcheck.1.md b/shellcheck.1.md index 92235f9..1cec34f 100644 --- a/shellcheck.1.md +++ b/shellcheck.1.md @@ -75,18 +75,19 @@ not warn at all, as `ksh` supports decimals in arithmetic contexts. directory, as in `-P SCRIPTDIR` or `-P SCRIPTDIR/../libs`. Subsequent `-P` flags accumulate and take predecence. +**-s**\ *shell*,\ **--shell=***shell* + +: Specify Bourne shell dialect. Valid values are *sh*, *bash*, *dash* and *ksh*. + The default is to deduce the shell from the file's `shell` directive, + shebang, or `.bash/.bats/.dash/.ksh` extension, in that order. *sh* refers to + POSIX `sh` (not the system's), and will warn of portability issues. + **-S**\ *SEVERITY*,\ **--severity=***severity* : Specify minimum severity of errors to consider. Valid values in order of severity are *error*, *warning*, *info*, *style* and *verbose*. The default is *style*. -**-s**\ *shell*,\ **--shell=***shell* - -: Specify Bourne shell dialect. Valid values are *sh*, *bash*, *dash* and *ksh*. - The default is to deduce the shell from the file's `shell` directive, - shebang, or `.bash/.bats/.dash/.ksh` extension, in that order. - **-V**,\ **--version** : Print version information and exit. From e2e65e135014616af05ee9eec1a0581b7566740a Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 29 Apr 2019 18:02:44 -0700 Subject: [PATCH 114/763] Warn about arithmetic base conversation in sh (fixes #1547) --- src/ShellCheck/Checks/ShellSupport.hs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index de9efc8..99cdd19 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -172,6 +172,8 @@ prop_checkBashisms88 = verifyNot checkBashisms "#!/bin/sh\nset -- wget -o foo 'h prop_checkBashisms89 = verifyNot checkBashisms "#!/bin/sh\nopts=$-\nset -\"$opts\"" prop_checkBashisms90 = verifyNot checkBashisms "#!/bin/sh\nset -o \"$opt\"" prop_checkBashisms91 = verify checkBashisms "#!/bin/sh\nwait -n" +prop_checkBashisms92 = verify checkBashisms "#!/bin/sh\necho $((16#FF))" +prop_checkBashisms93 = verify checkBashisms "#!/bin/sh\necho $(( 10#$(date +%m) ))" checkBashisms = ForShell [Sh, Dash] $ \t -> do params <- ask kludge params t @@ -385,6 +387,10 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do bashism t@(T_SourceCommand id src _) = let name = fromMaybe "" $ getCommandName src in when (name == "source") $ warnMsg id "'source' in place of '.' is" + bashism (TA_Expansion _ (T_Literal id str : _)) | str `matches` radix = + when (str `matches` radix) $ warnMsg id "arithmetic base conversion is" + where + radix = mkRegex "^[0-9]+#" bashism _ = return () varChars="_0-9a-zA-Z" From d72a5faa1f93e96671108ccc8628ba8798daf903 Mon Sep 17 00:00:00 2001 From: Virgil Date: Sat, 4 May 2019 16:27:44 +1000 Subject: [PATCH 115/763] :memo: docs: Update pandoc to match `Setup.hs` The sdist hook in [Setup.hs](Setup.hs) disables the `smart` extension when creating man page. --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d426216..9b0df56 100644 --- a/README.md +++ b/README.md @@ -210,8 +210,10 @@ or see the [storage bucket listing](https://shellcheck.storage.googleapis.com/in Distro packages already come with a `man` page. If you are building from source, it can be installed with: - pandoc -s -t man shellcheck.1.md -o shellcheck.1 - sudo mv shellcheck.1 /usr/share/man/man1 +```console +pandoc -s -f markdown-smart -t man shellcheck.1.md -o shellcheck.1 +sudo mv shellcheck.1 /usr/share/man/man1 +``` ## Travis CI From 37b24cc1296b50422540289ce243d3e894bbf6e9 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 4 May 2019 12:18:45 -0700 Subject: [PATCH 116/763] Don't warn about "a"b"c" in =~ regex (fixes #1565) --- src/ShellCheck/Analytics.hs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index f89df9a..ef05103 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1424,7 +1424,8 @@ prop_checkInexplicablyUnquoted5 = verifyNot checkInexplicablyUnquoted "\"$dir\"/ prop_checkInexplicablyUnquoted6 = verifyNot checkInexplicablyUnquoted "\"$dir\"some_stuff\"$file\"" prop_checkInexplicablyUnquoted7 = verifyNot checkInexplicablyUnquoted "${dir/\"foo\"/\"bar\"}" prop_checkInexplicablyUnquoted8 = verifyNot checkInexplicablyUnquoted " 'foo'\\\n 'bar'" -checkInexplicablyUnquoted _ (T_NormalWord id tokens) = mapM_ check (tails tokens) +prop_checkInexplicablyUnquoted9 = verifyNot checkInexplicablyUnquoted "[[ $x =~ \"foo\"(\"bar\"|\"baz\") ]]" +checkInexplicablyUnquoted params (T_NormalWord id tokens) = mapM_ check (tails tokens) where check (T_SingleQuoted _ _:T_Literal id str:_) | not (null str) && all isAlphaNum str = @@ -1435,12 +1436,21 @@ checkInexplicablyUnquoted _ (T_NormalWord id tokens) = mapM_ check (tails tokens T_DollarExpansion id _ -> warnAboutExpansion id T_DollarBraced id _ -> warnAboutExpansion id T_Literal id s -> - unless (quotesSingleThing a && quotesSingleThing b) $ + unless (quotesSingleThing a && quotesSingleThing b || isRegex (getPath (parentMap params) trapped)) $ warnAboutLiteral id _ -> return () check _ = return () + -- Regexes for [[ .. =~ re ]] are parsed with metacharacters like ()| as unquoted + -- literals, so avoid overtriggering on these. + isRegex t = + case t of + (T_Redirecting {} : _) -> False + (a:(TC_Binary _ _ "=~" lhs rhs):rest) -> getId a == getId rhs + _:rest -> isRegex rest + _ -> False + -- If the surrounding quotes quote single things, like "$foo"_and_then_some_"$stuff", -- the quotes were probably intentional and harmless. quotesSingleThing x = case x of From a3cd5979a2fedc249fc5dde4762ca9eae5e00159 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 4 May 2019 12:54:59 -0700 Subject: [PATCH 117/763] Update message for SC2171 --- src/ShellCheck/Analytics.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index ef05103..766efae 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2737,7 +2737,7 @@ checkTrailingBracket _ token = parameters = oversimplify command guard $ opposite `notElem` parameters return $ warn id 2171 $ - "Found trailing " ++ str ++ " outside test. Missing " ++ opposite ++ "?" + "Found trailing " ++ str ++ " outside test. Add missing " ++ opposite ++ " or quote if intentional." _ -> return () invert s = case s of From ec25fb4052047a60afc2ff13347ae8aa00a22fd9 Mon Sep 17 00:00:00 2001 From: Virgil Date: Sun, 5 May 2019 22:59:35 +1000 Subject: [PATCH 118/763] :memo: add Chocolatey installation method - [x] :sparkles: add Chocolatey for Windows installation - [x] :rotating_light: add language types to code blocks --- README.md | 48 +++++++++++++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index d426216..bac5561 100644 --- a/README.md +++ b/README.md @@ -21,12 +21,15 @@ See [the gallery of bad code](README.md#user-content-gallery-of-bad-code) for ex ## Table of Contents +- [Table of Contents](#table-of-contents) - [How to use](#how-to-use) - [On the web](#on-the-web) - [From your terminal](#from-your-terminal) - [In your editor](#in-your-editor) - [In your build or test suites](#in-your-build-or-test-suites) - [Installing](#installing) +- [Travis CI](#travis-ci) +- [Installing the shellcheck binary](#installing-the-shellcheck-binary) - [Compiling from source](#compiling-from-source) - [Installing Cabal](#installing-cabal) - [Compiling ShellCheck](#compiling-shellcheck) @@ -46,6 +49,7 @@ See [the gallery of bad code](README.md#user-content-gallery-of-bad-code) for ex - [Reporting bugs](#reporting-bugs) - [Contributing](#contributing) - [Copyright](#copyright) +- [Other Resources](#other-resources) ## How to use @@ -53,7 +57,7 @@ There are a number of ways to use ShellCheck! ### On the web -Paste a shell script on https://www.shellcheck.net for instant feedback. +Paste a shell script on for instant feedback. [ShellCheck.net](https://www.shellcheck.net) is always synchronized to the latest git commit, and is the easiest way to give ShellCheck a go. Tell your friends! @@ -88,7 +92,7 @@ It makes canonical use of exit codes, so you can just add a `shellcheck` command For example, in a Makefile: -``` +```Makefile check-scripts: # Fail if any of these files have warnings shellcheck myscripts/*.sh @@ -96,7 +100,7 @@ check-scripts: or in a Travis CI `.travis.yml` file: -``` +```yaml script: # Fail if any of these files have warnings - shellcheck myscripts/*.sh @@ -182,10 +186,18 @@ Or use OneClickInstall - https://software.opensuse.org/package/ShellCheck On Solus: eopkg install shellcheck - -On Windows (via [scoop](http://scoop.sh)): - scoop install shellcheck +On Windows (via [chocolatey](https://chocolatey.org/packages/shellcheck)): + +```cmd +C:\> choco install shellcheck +``` + +Or Windows (via [scoop](http://scoop.sh)): + +```cmd +C:\> scoop install shellcheck +``` From Snap Store: @@ -210,8 +222,10 @@ or see the [storage bucket listing](https://shellcheck.storage.googleapis.com/in Distro packages already come with a `man` page. If you are building from source, it can be installed with: - pandoc -s -t man shellcheck.1.md -o shellcheck.1 - sudo mv shellcheck.1 /usr/share/man/man1 +```console +pandoc -s -t man shellcheck.1.md -o shellcheck.1 +sudo mv shellcheck.1 /usr/share/man/man1 +``` ## Travis CI @@ -282,12 +296,16 @@ may use a legacy codepage. In `cmd.exe`, `powershell.exe` and Powershell ISE, make sure to use a TrueType font, not a Raster font, and set the active codepage to UTF-8 (65001) with `chcp`: - > chcp 65001 - Active code page: 65001 +```cmd +chcp 65001 +Active code page: 65001 +``` In Powershell ISE, you may need to additionally update the output encoding: - > [Console]::OutputEncoding = [System.Text.Encoding]::UTF8 +```powershell +[Console]::OutputEncoding = [System.Text.Encoding]::UTF8 +``` ### Running tests @@ -456,7 +474,7 @@ while getopts "a" f; do case $f in "b") # Unhandled getopts flags ## Testimonials -> At first you're like "shellcheck is awesome" but then you're like "wtf are we still using bash" +> 🔉 At first you're like "shellcheck is awesome" but then you're like "wtf are we still using bash" Alexander Tarasikov, [via Twitter](https://twitter.com/astarasikov/status/568825996532707330) @@ -465,13 +483,13 @@ Alexander Tarasikov, Issues can be ignored via environmental variable, command line, individually or globally within a file: -https://github.com/koalaman/shellcheck/wiki/Ignore + ## Reporting bugs Please use the GitHub issue tracker for any bugs or feature suggestions: -https://github.com/koalaman/shellcheck/issues + ## Contributing @@ -490,7 +508,7 @@ Copyright 2012-2018, Vidar 'koala_man' Holen and contributors. Happy ShellChecking! - ## Other Resources + * The wiki has [long form descriptions](https://github.com/koalaman/shellcheck/wiki/Checks) for each warning, e.g. [SC2221](https://github.com/koalaman/shellcheck/wiki/SC2221). * ShellCheck does not attempt to enforce any kind of formatting or indenting style, so also check out [shfmt](https://github.com/mvdan/sh)! From c0d4c5a1064b993cd418609e196743b68d8cfbd0 Mon Sep 17 00:00:00 2001 From: Eli Flanagan Date: Tue, 7 May 2019 07:48:43 -0400 Subject: [PATCH 119/763] ensure docker invocation is ephemeral Also adjust the tag to use the `:stable` tag mentioned in the prior line. --- README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index d426216..1da609f 100644 --- a/README.md +++ b/README.md @@ -114,14 +114,14 @@ Services and platforms with third party plugins: * [SonarQube](https://www.sonarqube.org/) through [sonar-shellcheck-plugin](https://github.com/emerald-squad/sonar-shellcheck-plugin) Most other services, including [GitLab](https://about.gitlab.com/), let you install -ShellCheck yourself, either through the system's package manager (see [Installing](#installing)), +ShellCheck yourself, either through the system's package manager (see [Installing](#installing)), or by downloading and unpacking a [binary release](#installing-the-shellcheck-binary). It's a good idea to manually install a specific ShellCheck version regardless. This avoids any surprise build breaks when a new version with new warnings is published. For customized filtering or reporting, ShellCheck can output simple JSON, CheckStyle compatible XML, -GCC compatible warnings as well as human readable text (with or without ANSI colors). See the +GCC compatible warnings as well as human readable text (with or without ANSI colors). See the [Integration](https://github.com/koalaman/shellcheck/wiki/Integration) wiki page for more documentation. ## Installing @@ -182,7 +182,7 @@ Or use OneClickInstall - https://software.opensuse.org/package/ShellCheck On Solus: eopkg install shellcheck - + On Windows (via [scoop](http://scoop.sh)): scoop install shellcheck @@ -195,7 +195,7 @@ From Docker Hub: ```sh docker pull koalaman/shellcheck:stable # Or :v0.4.7 for that version, or :latest for daily builds -docker run -v "$PWD:/mnt" koalaman/shellcheck myscript +docker run --rm -v "$PWD:/mnt" koalaman/shellcheck:stable myscript ``` or use `koalaman/shellcheck-alpine` if you want a larger Alpine Linux based image to extend. It works exactly like a regular Alpine image, but has shellcheck preinstalled. @@ -221,9 +221,9 @@ If you still want to do so in order to upgrade at your leisure or ensure the lat ## Installing the shellcheck binary -*Pre-requisite*: the program 'xz' needs to be installed on the system. -To install it on debian/ubuntu/linux mint, run `apt install xz-utils`. -To install it on Redhat/Fedora/CentOS, run `yum -y install xz`. +*Pre-requisite*: the program 'xz' needs to be installed on the system. +To install it on debian/ubuntu/linux mint, run `apt install xz-utils`. +To install it on Redhat/Fedora/CentOS, run `yum -y install xz`. ```bash export scversion="stable" # or "v0.4.7", or "latest" @@ -491,6 +491,6 @@ Copyright 2012-2018, Vidar 'koala_man' Holen and contributors. Happy ShellChecking! -## Other Resources +## Other Resources * The wiki has [long form descriptions](https://github.com/koalaman/shellcheck/wiki/Checks) for each warning, e.g. [SC2221](https://github.com/koalaman/shellcheck/wiki/SC2221). * ShellCheck does not attempt to enforce any kind of formatting or indenting style, so also check out [shfmt](https://github.com/mvdan/sh)! From 2521c1cf56d0f047eee6a7f716a9b99fe4d2e8a6 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 8 May 2019 18:03:58 -0700 Subject: [PATCH 120/763] Tweak README --- README.md | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index bac5561..f11c84e 100644 --- a/README.md +++ b/README.md @@ -28,8 +28,6 @@ See [the gallery of bad code](README.md#user-content-gallery-of-bad-code) for ex - [In your editor](#in-your-editor) - [In your build or test suites](#in-your-build-or-test-suites) - [Installing](#installing) -- [Travis CI](#travis-ci) -- [Installing the shellcheck binary](#installing-the-shellcheck-binary) - [Compiling from source](#compiling-from-source) - [Installing Cabal](#installing-cabal) - [Compiling ShellCheck](#compiling-shellcheck) @@ -227,22 +225,26 @@ pandoc -s -t man shellcheck.1.md -o shellcheck.1 sudo mv shellcheck.1 /usr/share/man/man1 ``` -## Travis CI +### Travis CI Travis CI has now integrated ShellCheck by default, so you don't need to manually install it. -If you still want to do so in order to upgrade at your leisure or ensure the latest release, follow the steps to install the shellcheck binary, bellow. +If you still want to do so in order to upgrade at your leisure or ensure you're +using the latest release, follow the steps below to install a binary version. -## Installing the shellcheck binary +### Installing a pre-compiled binary -*Pre-requisite*: the program 'xz' needs to be installed on the system. -To install it on debian/ubuntu/linux mint, run `apt install xz-utils`. -To install it on Redhat/Fedora/CentOS, run `yum -y install xz`. +The pre-compiled binaries come in `tar.xz` files. To decompress them, make sure +`xz` is installed. +On Debian/Ubuntu/Mint, you can `apt install xz-utils`. +On Redhat/Fedora/CentOS, `yum -y install xz`. + +A simple installer may do something like: ```bash -export scversion="stable" # or "v0.4.7", or "latest" -wget -qO- "https://storage.googleapis.com/shellcheck/shellcheck-"${scversion}".linux.x86_64.tar.xz" | tar -xJv -cp shellcheck-"${scversion}"/shellcheck /usr/bin/ +scversion="stable" # or "v0.4.7", or "latest" +wget -qO- "https://storage.googleapis.com/shellcheck/shellcheck-${scversion?}.linux.x86_64.tar.xz" | tar -xJv +cp "shellcheck-${scversion}/shellcheck" /usr/bin/ shellcheck --version ``` @@ -298,7 +300,6 @@ codepage to UTF-8 (65001) with `chcp`: ```cmd chcp 65001 -Active code page: 65001 ``` In Powershell ISE, you may need to additionally update the output encoding: @@ -474,7 +475,7 @@ while getopts "a" f; do case $f in "b") # Unhandled getopts flags ## Testimonials -> 🔉 At first you're like "shellcheck is awesome" but then you're like "wtf are we still using bash" +> At first you're like "shellcheck is awesome" but then you're like "wtf are we still using bash" Alexander Tarasikov, [via Twitter](https://twitter.com/astarasikov/status/568825996532707330) From d9e419d60fbd1507b573f33590a7f4b61841aa5b Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 9 May 2019 19:54:30 -0700 Subject: [PATCH 121/763] Add support for source-path directives (fixes #1577) --- CHANGELOG.md | 3 ++- shellcheck.1.md | 20 ++++++++++++++------ shellcheck.hs | 4 ++-- src/ShellCheck/AST.hs | 1 + src/ShellCheck/Checker.hs | 22 +++++++++++++++++++--- src/ShellCheck/Interface.hs | 10 +++++++--- src/ShellCheck/Parser.hs | 24 ++++++++++++++++++++++-- 7 files changed, 67 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 79f7910..9cad51f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,8 @@ - Files containing Bats tests can now be checked - Directory wide directives can now be placed in a `.shellcheckrc` - Verbose mode: Use `-S verbose` for especially pedantic suggestions -- Source paths: Use `-P dir1:dir2` to specify path for sourced files +- Source paths: Use `-P dir1:dir2` or a `source-path=dir1` directive + to specify search paths for sourced files. - SC2249: Warn about `case` with missing default case (verbose) - SC2248: Warn about unquoted variables without special chars (verbose) - SC2247: Warn about $"(cmd)" and $"{var}" diff --git a/shellcheck.1.md b/shellcheck.1.md index 1cec34f..220cc04 100644 --- a/shellcheck.1.md +++ b/shellcheck.1.md @@ -70,10 +70,8 @@ not warn at all, as `ksh` supports decimals in arithmetic contexts. **-P**\ *SOURCEPATH*,\ **--source-path=***SOURCEPATH* : Specify paths to search for sourced files, separated by `:` on Unix and - `;` on Windows. Absolute paths will also be rooted in these. The special - path `SCRIPTDIR` can be used to specify the currently checked script's - directory, as in `-P SCRIPTDIR` or `-P SCRIPTDIR/../libs`. Subsequent - `-P` flags accumulate and take predecence. + `;` on Windows. This is equivalent to specifying `search-path` + directives. **-s**\ *shell*,\ **--shell=***shell* @@ -201,6 +199,14 @@ Valid keys are: used to tell shellcheck where to look for a file whose name is determined at runtime, or to skip a source by telling it to use `/dev/null`. +**source-path** +: Add a directory to the search path for `source`/`.` statements (by default, + only ShellCheck's working directory is included). Absolute paths will also + be rooted in these paths. The special path `SCRIPTDIR` can be used to + specify the currently checked script's directory, as in + `source-path=SCRIPTDIR` or `source-path=SCRIPTDIR/../libs`. Multiple + paths accumulate, and `-P` takes precedence over them. + **shell** : Overrides the shell detected from the shebang. This is useful for files meant to be included (and thus lacking a shebang), or possibly @@ -213,8 +219,10 @@ it will read `key=value` pairs from it and treat them as file-wide directives. Here is an example `.shellcheckrc`: - # Don't suggest using -n in [ $var ] - disable=SC2244 + # Look for 'source'd files relative to the checked script, + # and also look for absolute paths in /mnt/chroot + source-path=SCRIPTDIR + source-path=/mnt/chroot # Allow using `which` since it gives full paths and is common enough disable=SC2230 diff --git a/shellcheck.hs b/shellcheck.hs index ac6639a..137aaba 100644 --- a/shellcheck.hs +++ b/shellcheck.hs @@ -471,7 +471,7 @@ ioInterface options files = do first <- a arg if not first then return False else b arg - findSourceFile inputs sourcePaths currentScript original = + findSourceFile inputs sourcePathFlag currentScript sourcePathAnnotation original = if isAbsolute original then let (_, relative) = splitDrive original @@ -481,7 +481,7 @@ ioInterface options files = do where find filename deflt = do sources <- filterM ((allowable inputs) `andM` doesFileExist) - (map ( filename) $ map adjustPath sourcePaths) + (map ( filename) $ map adjustPath $ sourcePathFlag ++ sourcePathAnnotation) case sources of [] -> return deflt (first:_) -> return first diff --git a/src/ShellCheck/AST.hs b/src/ShellCheck/AST.hs index aedb148..9ec892d 100644 --- a/src/ShellCheck/AST.hs +++ b/src/ShellCheck/AST.hs @@ -146,6 +146,7 @@ data Annotation = DisableComment Integer | SourceOverride String | ShellOverride String + | SourcePath String deriving (Show, Eq) data ConditionType = DoubleBracket | SingleBracket deriving (Show, Eq) diff --git a/src/ShellCheck/Checker.hs b/src/ShellCheck/Checker.hs index b6ee068..e73636d 100644 --- a/src/ShellCheck/Checker.hs +++ b/src/ShellCheck/Checker.hs @@ -215,6 +215,7 @@ prop_worksWhenSourcingWithDashDash = prop_worksWhenDotting = null $ checkWithIncludes [("lib", "bar=1")] ". lib; echo \"$bar\"" +-- FIXME: This should really be giving [1093], "recursively sourced" prop_noInfiniteSourcing = [] == checkWithIncludes [("lib", "source lib")] "source lib" @@ -236,6 +237,12 @@ prop_recursiveAnalysis = prop_recursiveParsing = [1037] == checkRecursive [("lib", "echo \"$10\"")] "source lib" +prop_nonRecursiveAnalysis = + [] == checkWithIncludes [("lib", "echo $1")] "source lib" + +prop_nonRecursiveParsing = + [] == checkWithIncludes [("lib", "echo \"$10\"")] "source lib" + prop_sourceDirectiveDoesntFollowFile = null $ checkWithIncludes [("foo", "source bar"), ("bar", "baz=3")] @@ -342,17 +349,26 @@ prop_brokenRcGetsWarning = result == [1134, 2086] prop_sourcePathRedirectsName = result == [2086] where - f "dir/myscript" "lib" = return "foo/lib" + f "dir/myscript" _ "lib" = return "foo/lib" result = checkWithIncludesAndSourcePath [("foo/lib", "echo $1")] f emptyCheckSpec { csScript = "#!/bin/bash\nsource lib", csFilename = "dir/myscript", csCheckSourced = True } +prop_sourcePathAddsAnnotation = result == [2086] + where + f "dir/myscript" ["mypath"] "lib" = return "foo/lib" + result = checkWithIncludesAndSourcePath [("foo/lib", "echo $1")] f emptyCheckSpec { + csScript = "#!/bin/bash\n# shellcheck source-path=mypath\nsource lib", + csFilename = "dir/myscript", + csCheckSourced = True + } + prop_sourcePathRedirectsDirective = result == [2086] where - f "dir/myscript" "lib" = return "foo/lib" - f _ _ = return "/dev/null" + f "dir/myscript" _ "lib" = return "foo/lib" + f _ _ _ = return "/dev/null" result = checkWithIncludesAndSourcePath [("foo/lib", "echo $1")] f emptyCheckSpec { csScript = "#!/bin/bash\n# shellcheck source=lib\nsource kittens", csFilename = "dir/myscript", diff --git a/src/ShellCheck/Interface.hs b/src/ShellCheck/Interface.hs index fa342e2..1d0cc6b 100644 --- a/src/ShellCheck/Interface.hs +++ b/src/ShellCheck/Interface.hs @@ -73,8 +73,12 @@ import qualified Data.Map as Map data SystemInterface m = SystemInterface { -- Read a file by filename, or return an error siReadFile :: String -> m (Either ErrorMessage String), - -- Given the current script and a sourced file, find the sourced file - siFindSource :: String -> String -> m FilePath, + -- Given: + -- the current script, + -- a list of source-path annotations in effect, + -- and a sourced file, + -- find the sourced file + siFindSource :: String -> [String] -> String -> m FilePath, -- Get the configuration file (name, contents) for a filename siGetConfig :: String -> m (Maybe (FilePath, String)) } @@ -297,7 +301,7 @@ mockedSystemInterface files = SystemInterface { case filter ((== file) . fst) files of [] -> return $ Left "File not included in mock." [(_, contents)] -> return $ Right contents - fs _ file = return file + fs _ _ file = return file mockRcFile rcfile mock = mock { siGetConfig = const . return $ Just (".shellcheckrc", rcfile) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 75e45ce..91bc3f1 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -264,6 +264,15 @@ shouldIgnoreCode code = do disabling' (DisableComment n) = code == n disabling' _ = False +getCurrentAnnotations includeSource = + concatMap get . takeWhile (not . isBoundary) <$> getCurrentContexts + where + get (ContextAnnotation list) = list + get _ = [] + isBoundary (ContextSource _) = not includeSource + isBoundary _ = False + + shouldFollow file = do context <- getCurrentContexts if any isThisFile context @@ -966,7 +975,7 @@ readAnnotationWithoutPrefix = do where readKey = do keyPos <- getPosition - key <- many1 letter + key <- many1 (letter <|> char '-') char '=' <|> fail "Expected '=' after directive key" annotations <- case key of "disable" -> readCode `sepBy` char ',' @@ -980,6 +989,10 @@ readAnnotationWithoutPrefix = do filename <- many1 $ noneOf " \n" return [SourceOverride filename] + "source-path" -> do + dirname <- many1 $ noneOf " \n" + return [SourcePath dirname] + "shell" -> do pos <- getPosition shell <- many1 $ noneOf " \n" @@ -2079,6 +2092,7 @@ readSource t@(T_Redirecting _ _ (T_SimpleCommand cmdId _ (cmd:file':rest'))) = d proceed <- shouldFollow filename if not proceed then do + -- FIXME: This actually gets squashed without -a parseNoteAtId (getId file) InfoC 1093 "This file appears to be recursively sourced. Ignoring." return t @@ -2089,7 +2103,8 @@ readSource t@(T_Redirecting _ _ (T_SimpleCommand cmdId _ (cmd:file':rest'))) = d then return (Right "") else do currentScript <- Mr.asks currentFilename - filename' <- system $ siFindSource sys currentScript filename + paths <- mapMaybe getSourcePath <$> getCurrentAnnotations True + filename' <- system $ siFindSource sys currentScript paths filename system $ siReadFile sys filename' case input of Left err -> do @@ -2118,6 +2133,11 @@ readSource t@(T_Redirecting _ _ (T_SimpleCommand cmdId _ (cmd:file':rest'))) = d x -> file getFile file _ = file + getSourcePath t = + case t of + SourcePath x -> Just x + _ -> Nothing + subRead name script = withContext (ContextSource name) $ inSeparateContext $ From bb63d66f7c837eb76236c7026a7a03766928adc2 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 9 May 2019 20:17:35 -0700 Subject: [PATCH 122/763] Delete trailing spaces --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 92a27dc..60549bf 100644 --- a/README.md +++ b/README.md @@ -116,14 +116,14 @@ Services and platforms with third party plugins: * [SonarQube](https://www.sonarqube.org/) through [sonar-shellcheck-plugin](https://github.com/emerald-squad/sonar-shellcheck-plugin) Most other services, including [GitLab](https://about.gitlab.com/), let you install -ShellCheck yourself, either through the system's package manager (see [Installing](#installing)), +ShellCheck yourself, either through the system's package manager (see [Installing](#installing)), or by downloading and unpacking a [binary release](#installing-the-shellcheck-binary). It's a good idea to manually install a specific ShellCheck version regardless. This avoids any surprise build breaks when a new version with new warnings is published. For customized filtering or reporting, ShellCheck can output simple JSON, CheckStyle compatible XML, -GCC compatible warnings as well as human readable text (with or without ANSI colors). See the +GCC compatible warnings as well as human readable text (with or without ANSI colors). See the [Integration](https://github.com/koalaman/shellcheck/wiki/Integration) wiki page for more documentation. ## Installing @@ -509,7 +509,7 @@ Copyright 2012-2018, Vidar 'koala_man' Holen and contributors. Happy ShellChecking! -## Other Resources +## Other Resources * The wiki has [long form descriptions](https://github.com/koalaman/shellcheck/wiki/Checks) for each warning, e.g. [SC2221](https://github.com/koalaman/shellcheck/wiki/SC2221). * ShellCheck does not attempt to enforce any kind of formatting or indenting style, so also check out [shfmt](https://github.com/mvdan/sh)! From 5b177d62cbe1816f08ce95504d90814727f4a646 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 9 May 2019 20:22:10 -0700 Subject: [PATCH 123/763] Simplify docker instructions --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8cb5fc3..a88aa31 100644 --- a/README.md +++ b/README.md @@ -204,8 +204,8 @@ From Snap Store: From Docker Hub: ```sh -docker pull koalaman/shellcheck:stable # Or :v0.4.7 for that version, or :latest for daily builds -docker run --rm -v "$PWD:/mnt" koalaman/shellcheck myscript +docker run --rm -v "$PWD:/mnt" koalaman/shellcheck:stable myscript +# Or :v0.4.7 for that version, or :latest for daily builds ``` or use `koalaman/shellcheck-alpine` if you want a larger Alpine Linux based image to extend. It works exactly like a regular Alpine image, but has shellcheck preinstalled. From 58205a3573e8d82bf2a0b6926f96062b66852892 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 12 May 2019 15:49:52 -0700 Subject: [PATCH 124/763] Emit resolved rather than apparent filename for 'source' (fixes #1579) --- src/ShellCheck/Parser.hs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 91bc3f1..f5a52ed 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -2098,14 +2098,15 @@ readSource t@(T_Redirecting _ _ (T_SimpleCommand cmdId _ (cmd:file':rest'))) = d return t else do sys <- Mr.asks systemInterface - input <- + (input, resolvedFile) <- if filename == "/dev/null" -- always allow /dev/null - then return (Right "") + then return (Right "", filename) else do currentScript <- Mr.asks currentFilename paths <- mapMaybe getSourcePath <$> getCurrentAnnotations True - filename' <- system $ siFindSource sys currentScript paths filename - system $ siReadFile sys filename' + resolved <- system $ siFindSource sys currentScript paths filename + contents <- system $ siReadFile sys resolved + return (contents, resolved) case input of Left err -> do parseNoteAtId (getId file) InfoC 1091 $ @@ -2116,7 +2117,7 @@ readSource t@(T_Redirecting _ _ (T_SimpleCommand cmdId _ (cmd:file':rest'))) = d id2 <- getNewIdFor cmdId let included = do - src <- subRead filename script + src <- subRead resolvedFile script return $ T_SourceCommand id1 t (T_Include id2 src) let failed = do From 5fb1da6814289f7f0a272b9eef499c39632a3cd4 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 12 May 2019 19:14:04 -0700 Subject: [PATCH 125/763] Replace verbose checks with optional checks --- CHANGELOG.md | 3 +- shellcheck.1.md | 25 ++++- shellcheck.hs | 33 +++++- src/ShellCheck/AST.hs | 1 + src/ShellCheck/Analytics.hs | 155 ++++++++++++++++++++--------- src/ShellCheck/Analyzer.hs | 6 +- src/ShellCheck/AnalyzerLib.hs | 27 +++-- src/ShellCheck/Checker.hs | 17 +++- src/ShellCheck/Formatter/Format.hs | 1 - src/ShellCheck/Interface.hs | 31 ++++-- src/ShellCheck/Parser.hs | 4 + 11 files changed, 229 insertions(+), 74 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9cad51f..bb623f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,8 @@ - Preliminary support for fix suggestions - Files containing Bats tests can now be checked - Directory wide directives can now be placed in a `.shellcheckrc` -- Verbose mode: Use `-S verbose` for especially pedantic suggestions +- Optional checks: Use `--list-optional` to show a list of tests, + Enable with `-o` flags or `enable=name` directives - Source paths: Use `-P dir1:dir2` or a `source-path=dir1` directive to specify search paths for sourced files. - SC2249: Warn about `case` with missing default case (verbose) diff --git a/shellcheck.1.md b/shellcheck.1.md index 220cc04..f8cbd44 100644 --- a/shellcheck.1.md +++ b/shellcheck.1.md @@ -63,10 +63,21 @@ not warn at all, as `ksh` supports decimals in arithmetic contexts. standard output. Subsequent **-f** options are ignored, see **FORMATS** below for more information. +**--list-optional** + +: Output a list of known optional checks. These can be enabled with **-o** + flags or **enable** directives. + **--norc** : Don't try to look for .shellcheckrc configuration files. +**-o**\ *NAME1*[,*NAME2*...],\ **--enable=***NAME1*[,*NAME2*...] + +: Enable optional checks. The special name *all* enables all of them. + Subsequent **-o** options accumulate. This is equivalent to specifying + **enable** directives. + **-P**\ *SOURCEPATH*,\ **--source-path=***SOURCEPATH* : Specify paths to search for sourced files, separated by `:` on Unix and @@ -83,7 +94,7 @@ not warn at all, as `ksh` supports decimals in arithmetic contexts. **-S**\ *SEVERITY*,\ **--severity=***severity* : Specify minimum severity of errors to consider. Valid values in order of - severity are *error*, *warning*, *info*, *style* and *verbose*. + severity are *error*, *warning*, *info* and *style*. The default is *style*. **-V**,\ **--version** @@ -163,8 +174,9 @@ not warn at all, as `ksh` supports decimals in arithmetic contexts. # DIRECTIVES -ShellCheck directives can be specified as comments in the shell script -before a command or block: +ShellCheck directives can be specified as comments in the shell script. +If they appear before the first command, they are considered file-wide. +Otherwise, they apply to the immediately following command or block: # shellcheck key=value key=value command-or-structure @@ -194,6 +206,10 @@ Valid keys are: The command can be a simple command like `echo foo`, or a compound command like a function definition, subshell block or loop. +**enable** +: Enable an optional check by name, as listed with **--list-optional**. + Only file-wide `enable` directives are considered. + **source** : Overrides the filename included by a `source`/`.` statement. This can be used to tell shellcheck where to look for a file whose name is determined @@ -224,6 +240,9 @@ Here is an example `.shellcheckrc`: source-path=SCRIPTDIR source-path=/mnt/chroot + # Turn on warnings for unquoted variables with safe values + enable=quote-safe-variables + # Allow using `which` since it gives full paths and is common enough disable=SC2230 diff --git a/shellcheck.hs b/shellcheck.hs index 137aaba..c8ea7fb 100644 --- a/shellcheck.hs +++ b/shellcheck.hs @@ -17,6 +17,7 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . -} +import qualified ShellCheck.Analyzer import ShellCheck.Checker import ShellCheck.Data import ShellCheck.Interface @@ -98,8 +99,13 @@ options = [ Option "f" ["format"] (ReqArg (Flag "format") "FORMAT") $ "Output format (" ++ formatList ++ ")", + Option "" ["list-optional"] + (NoArg $ Flag "list-optional" "true") "List checks disabled by default", Option "" ["norc"] (NoArg $ Flag "norc" "true") "Don't look for .shellcheckrc files", + Option "o" ["enable"] + (ReqArg (Flag "enable") "check1,check2..") + "List of optional checks to enable (or 'all')", Option "P" ["source-path"] (ReqArg (Flag "source-path") "SOURCEPATHS") "Specify path when looking for sourced files (\"SCRIPTDIR\" for script's dir)", @@ -108,7 +114,7 @@ options = [ "Specify dialect (sh, bash, dash, ksh)", Option "S" ["severity"] (ReqArg (Flag "severity") "SEVERITY") - "Minimum severity of errors to consider (error, warning, info, style, verbose)", + "Minimum severity of errors to consider (error, warning, info, style)", Option "V" ["version"] (NoArg $ Flag "version" "true") "Print version information", Option "W" ["wiki-link-count"] @@ -259,8 +265,7 @@ parseSeverityOption value = ("error", ErrorC), ("warning", WarningC), ("info", InfoC), - ("style", StyleC), - ("verbose", VerboseC) + ("style", StyleC) ] parseOption flag options = @@ -299,6 +304,10 @@ parseOption flag options = liftIO printVersion throwError NoProblems + Flag "list-optional" _ -> do + liftIO printOptional + throwError NoProblems + Flag "help" _ -> do liftIO $ putStrLn getUsageInfo throwError NoProblems @@ -352,6 +361,13 @@ parseOption flag options = } } + Flag "enable" value -> + let cs = checkSpec options in return options { + checkSpec = cs { + csOptionalChecks = (csOptionalChecks cs) ++ split ',' value + } + } + -- This flag is handled specially in 'process' Flag "format" _ -> return options @@ -547,3 +563,14 @@ printVersion = do putStrLn $ "version: " ++ shellcheckVersion putStrLn "license: GNU General Public License, version 3" putStrLn "website: https://www.shellcheck.net" + +printOptional = do + mapM f list + where + list = sortOn cdName ShellCheck.Analyzer.optionalChecks + f item = do + putStrLn $ "name: " ++ cdName item + putStrLn $ "desc: " ++ cdDescription item + putStrLn $ "example: " ++ cdPositive item + putStrLn $ "fix: " ++ cdNegative item + putStrLn "" diff --git a/src/ShellCheck/AST.hs b/src/ShellCheck/AST.hs index 9ec892d..5b4254f 100644 --- a/src/ShellCheck/AST.hs +++ b/src/ShellCheck/AST.hs @@ -144,6 +144,7 @@ data Token = data Annotation = DisableComment Integer + | EnableComment String | SourceOverride String | ShellOverride String | SourcePath String diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 766efae..ac8fcdb 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -19,7 +19,7 @@ -} {-# LANGUAGE TemplateHaskell #-} {-# LANGUAGE FlexibleContexts #-} -module ShellCheck.Analytics (runAnalytics, ShellCheck.Analytics.runTests) where +module ShellCheck.Analytics (runAnalytics, optionalChecks, ShellCheck.Analytics.runTests) where import ShellCheck.AST import ShellCheck.ASTLib @@ -49,11 +49,9 @@ import Test.QuickCheck.Test (quickCheckWithResult, stdArgs, maxSuccess) -- Checks that are run on the AST root treeChecks :: [Parameters -> Token -> [TokenComment]] treeChecks = [ - runNodeAnalysis - (\p t -> (mapM_ ((\ f -> f t) . (\ f -> f p)) - nodeChecks)) + nodeChecksToTreeCheck nodeChecks ,subshellAssignmentCheck - ,checkVerboseSpacefulness + ,checkSpacefulness ,checkQuotesInLiterals ,checkShebangParameters ,checkFunctionsUsedExternally @@ -69,7 +67,14 @@ treeChecks = [ runAnalytics :: AnalysisSpec -> [TokenComment] runAnalytics options = - runList options treeChecks + runList options treeChecks ++ runList options optionalChecks + where + root = asScript options + optionals = getEnableDirectives root ++ asOptionalChecks options + optionalChecks = + if "all" `elem` optionals + then map snd optionalTreeChecks + else mapMaybe (\c -> Map.lookup c optionalCheckMap) optionals runList :: AnalysisSpec -> [Parameters -> Token -> [TokenComment]] -> [TokenComment] @@ -79,13 +84,27 @@ runList spec list = notes params = makeParameters spec notes = concatMap (\f -> f params root) list +getEnableDirectives root = + case root of + T_Annotation _ list _ -> mapMaybe getEnable list + _ -> [] + where + getEnable t = + case t of + EnableComment s -> return s + _ -> Nothing checkList l t = concatMap (\f -> f t) l - -- Checks that are run on each node in the AST runNodeAnalysis f p t = execWriter (doAnalysis (f p) t) +-- Perform multiple node checks in a single iteration over the tree +nodeChecksToTreeCheck checkList = + runNodeAnalysis + (\p t -> (mapM_ ((\ f -> f t) . (\ f -> f p)) + checkList)) + nodeChecks :: [Parameters -> Token -> Writer [TokenComment] ()] nodeChecks = [ checkUuoc @@ -170,11 +189,46 @@ nodeChecks = [ ,checkSubshelledTests ,checkInvertedStringTest ,checkRedirectionToCommand - ,checkNullaryExpansionTest ,checkDollarQuoteParen - ,checkDefaultCase ] +optionalChecks = map fst optionalTreeChecks + + +prop_verifyOptionalExamples = all check optionalTreeChecks + where + check (desc, check) = + verifyTree check (cdPositive desc) + && verifyNotTree check (cdNegative desc) + +optionalTreeChecks :: [(CheckDescription, (Parameters -> Token -> [TokenComment]))] +optionalTreeChecks = [ + (newCheckDescription { + cdName = "quote-safe-variables", + cdDescription = "Suggest quoting variables without metacharacters", + cdPositive = "var=hello; echo $var", + cdNegative = "var=hello; echo \"$var\"" + }, checkVerboseSpacefulness) + + ,(newCheckDescription { + cdName = "avoid-nullary-conditions", + cdDescription = "Suggest explicitly using -n in `[ $var ]`", + cdPositive = "[ \"$var\" ]", + cdNegative = "[ -n \"$var\" ]" + }, nodeChecksToTreeCheck [checkNullaryExpansionTest]) + + ,(newCheckDescription { + cdName = "add-default-case", + cdDescription = "Suggest adding a default case in `case` statements", + cdPositive = "case $? in 0) echo 'Success';; esac", + cdNegative = "case $? in 0) echo 'Success';; *) echo 'Fail' ;; esac" + }, nodeChecksToTreeCheck [checkDefaultCase]) + ] + +optionalCheckMap :: Map.Map String (Parameters -> Token -> [TokenComment]) +optionalCheckMap = Map.fromList $ map item optionalTreeChecks + where + item (desc, check) = (cdName desc, check) wouldHaveBeenGlob s = '*' `elem` s @@ -1650,12 +1704,10 @@ prop_checkSpacefulness2 = verifyNotTree checkSpacefulness "a='cow moo'; [[ $a ]] prop_checkSpacefulness3 = verifyNotTree checkSpacefulness "a='cow*.mp3'; echo \"$a\"" prop_checkSpacefulness4 = verifyTree checkSpacefulness "for f in *.mp3; do echo $f; done" prop_checkSpacefulness4a= verifyNotTree checkSpacefulness "foo=3; foo=$(echo $foo)" -prop_checkSpacefulness4v= verifyTree checkVerboseSpacefulness "foo=3; foo=$(echo $foo)" prop_checkSpacefulness5 = verifyTree checkSpacefulness "a='*'; b=$a; c=lol${b//foo/bar}; echo $c" prop_checkSpacefulness6 = verifyTree checkSpacefulness "a=foo$(lol); echo $a" prop_checkSpacefulness7 = verifyTree checkSpacefulness "a=foo\\ bar; rm $a" prop_checkSpacefulness8 = verifyNotTree checkSpacefulness "a=foo\\ bar; a=foo; rm $a" -prop_checkSpacefulness8v= verifyTree checkVerboseSpacefulness "a=foo\\ bar; a=foo; rm $a" prop_checkSpacefulness10= verifyTree checkSpacefulness "rm $1" prop_checkSpacefulness11= verifyTree checkSpacefulness "rm ${10//foo/bar}" prop_checkSpacefulness12= verifyNotTree checkSpacefulness "(( $1 + 3 ))" @@ -1675,7 +1727,6 @@ prop_checkSpacefulness25= verifyTree checkSpacefulness "a='s/[0-9]//g'; sed $a" prop_checkSpacefulness26= verifyTree checkSpacefulness "a='foo bar'; echo {1,2,$a}" prop_checkSpacefulness27= verifyNotTree checkSpacefulness "echo ${a:+'foo'}" prop_checkSpacefulness28= verifyNotTree checkSpacefulness "exec {n}>&1; echo $n" -prop_checkSpacefulness28v = verifyTree checkVerboseSpacefulness "exec {n}>&1; echo $n" prop_checkSpacefulness29= verifyNotTree checkSpacefulness "n=$(stuff); exec {n}>&-;" prop_checkSpacefulness30= verifyTree checkSpacefulness "file='foo bar'; echo foo > $file;" prop_checkSpacefulness31= verifyNotTree checkSpacefulness "echo \"`echo \\\"$1\\\"`\"" @@ -1684,22 +1735,53 @@ prop_checkSpacefulness33= verifyTree checkSpacefulness "for file; do echo $file; prop_checkSpacefulness34= verifyTree checkSpacefulness "declare foo$n=$1" prop_checkSpacefulness35= verifyNotTree checkSpacefulness "echo ${1+\"$1\"}" prop_checkSpacefulness36= verifyNotTree checkSpacefulness "arg=$#; echo $arg" -prop_checkSpacefulness36v = verifyTree checkVerboseSpacefulness "arg=$#; echo $arg" prop_checkSpacefulness37= verifyNotTree checkSpacefulness "@test 'status' {\n [ $status -eq 0 ]\n}" prop_checkSpacefulness37v = verifyTree checkVerboseSpacefulness "@test 'status' {\n [ $status -eq 0 ]\n}" --- This is slightly awkward because we want the tests to --- discriminate between normal and verbose output. -checkSpacefulness params t = checkSpacefulness' False params t -checkVerboseSpacefulness params t = checkSpacefulness' True params t -checkSpacefulness' alsoVerbose params t = +-- This is slightly awkward because we want to support structured +-- optional checks based on nearly the same logic +checkSpacefulness params = checkSpacefulness' onFind params + where + emit x = tell [x] + onFind spaces token _ = + when spaces $ + if isDefaultAssignment (parentMap params) token + then + emit $ makeComment InfoC (getId token) 2223 + "This default assignment may cause DoS due to globbing. Quote it." + else + emit $ makeCommentWithFix InfoC (getId token) 2086 + "Double quote to prevent globbing and word splitting." + (addDoubleQuotesAround params token) + + isDefaultAssignment parents token = + let modifier = getBracedModifier $ bracedString token in + any (`isPrefixOf` modifier) ["=", ":="] + && isParamTo parents ":" token + + +prop_checkSpacefulness4v= verifyTree checkVerboseSpacefulness "foo=3; foo=$(echo $foo)" +prop_checkSpacefulness8v= verifyTree checkVerboseSpacefulness "a=foo\\ bar; a=foo; rm $a" +prop_checkSpacefulness28v = verifyTree checkVerboseSpacefulness "exec {n}>&1; echo $n" +prop_checkSpacefulness36v = verifyTree checkVerboseSpacefulness "arg=$#; echo $arg" +checkVerboseSpacefulness params = checkSpacefulness' onFind params + where + onFind spaces token name = + when (not spaces && name `notElem` specialVariablesWithoutSpaces) $ + tell [makeCommentWithFix StyleC (getId token) 2248 + "Prefer double quoting even when variables don't contain special characters." + (addDoubleQuotesAround params token)] + +addDoubleQuotesAround params token = (surroundWidth (getId token) params "\"") +checkSpacefulness' + :: (Bool -> Token -> String -> Writer [TokenComment] ()) -> + Parameters -> Token -> [TokenComment] +checkSpacefulness' onFind params t = doVariableFlowAnalysis readF writeF (Map.fromList defaults) (variableFlow params) where defaults = zip variablesWithoutSpaces (repeat False) - hasSpaces name = do - map <- get - return $ Map.findWithDefault True name map + hasSpaces name = gets (Map.findWithDefault True name) setSpaces name bool = modify $ Map.insert name bool @@ -1714,24 +1796,9 @@ checkSpacefulness' alsoVerbose params t = && not (isQuotedAlternativeReference token) && not (usedAsCommandName parents token) - return . execWriter $ when needsQuoting $ - if spaces - then - if isDefaultAssignment (parentMap params) token - then - emit $ makeComment InfoC (getId token) 2223 - "This default assignment may cause DoS due to globbing. Quote it." - else - emit $ makeCommentWithFix InfoC (getId token) 2086 - "Double quote to prevent globbing and word splitting." - (fixFor token) - else - when (alsoVerbose && name `notElem` specialVariablesWithoutSpaces) $ - emit $ makeCommentWithFix VerboseC (getId token) 2248 - "Prefer double quoting even when variables don't contain special characters." - (fixFor token) + return . execWriter $ when needsQuoting $ onFind spaces token name + where - fixFor token = (surroundWidth (getId token) params "\"") emit x = tell [x] writeF _ _ name (DataString SourceExternal) = setSpaces name True >> return [] @@ -1771,12 +1838,6 @@ checkSpacefulness' alsoVerbose params t = globspace = "*?[] \t\n" containsAny s = any (`elem` s) - isDefaultAssignment parents token = - let modifier = getBracedModifier $ bracedString token in - isExpansion token - && any (`isPrefixOf` modifier) ["=", ":="] - && isParamTo parents ":" token - prop_checkQuotesInLiterals1 = verifyTree checkQuotesInLiterals "param='--foo=\"bar\"'; app $param" prop_checkQuotesInLiterals1a= verifyTree checkQuotesInLiterals "param=\"--foo='lolbar'\"; app $param" prop_checkQuotesInLiterals2 = verifyNotTree checkQuotesInLiterals "param='--foo=\"bar\"'; app \"$param\"" @@ -3224,11 +3285,11 @@ checkNullaryExpansionTest params t = TC_Nullary _ _ word -> case getWordParts word of [t] | isCommandSubstitution t -> - verboseWithFix id 2243 "Prefer explicit -n to check for output (or run command without [/[[ to check for success)." fix + styleWithFix id 2243 "Prefer explicit -n to check for output (or run command without [/[[ to check for success)." fix -- If they're constant, you get SC2157 &co x | all (not . isConstant) x -> - verboseWithFix id 2244 "Prefer explicit -n to check non-empty string (or use =/-ne to check boolean/integer)." fix + styleWithFix id 2244 "Prefer explicit -n to check non-empty string (or use =/-ne to check boolean/integer)." fix _ -> return () where id = getId word @@ -3256,7 +3317,7 @@ checkDefaultCase _ t = case t of T_CaseExpression id _ list -> unless (any canMatchAny list) $ - verbose id 2249 "Consider adding a default *) case, even if it just exits with error." + info id 2249 "Consider adding a default *) case, even if it just exits with error." _ -> return () where canMatchAny (_, list, _) = any canMatchAny' list diff --git a/src/ShellCheck/Analyzer.hs b/src/ShellCheck/Analyzer.hs index ffbc4e5..442daba 100644 --- a/src/ShellCheck/Analyzer.hs +++ b/src/ShellCheck/Analyzer.hs @@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . -} -module ShellCheck.Analyzer (analyzeScript) where +module ShellCheck.Analyzer (analyzeScript, ShellCheck.Analyzer.optionalChecks) where import ShellCheck.Analytics import ShellCheck.AnalyzerLib @@ -43,3 +43,7 @@ checkers params = mconcat $ map ($ params) [ ShellCheck.Checks.Commands.checker, ShellCheck.Checks.ShellSupport.checker ] + +optionalChecks = mconcat $ [ + ShellCheck.Analytics.optionalChecks + ] diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 01fcc8f..5783820 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -77,14 +77,22 @@ composeAnalyzers :: (a -> Analysis) -> (a -> Analysis) -> a -> Analysis composeAnalyzers f g x = f x >> g x data Parameters = Parameters { - hasLastpipe :: Bool, -- Whether this script has the 'lastpipe' option set/default. - hasSetE :: Bool, -- Whether this script has 'set -e' anywhere. - variableFlow :: [StackData], -- A linear (bad) analysis of data flow - parentMap :: Map.Map Id Token, -- A map from Id to parent Token - shellType :: Shell, -- The shell type, such as Bash or Ksh - shellTypeSpecified :: Bool, -- True if shell type was forced via flags - rootNode :: Token, -- The root node of the AST - tokenPositions :: Map.Map Id (Position, Position) -- map from token id to start and end position + -- Whether this script has the 'lastpipe' option set/default. + hasLastpipe :: Bool, + -- Whether this script has 'set -e' anywhere. + hasSetE :: Bool, + -- A linear (bad) analysis of data flow + variableFlow :: [StackData], + -- A map from Id to parent Token + parentMap :: Map.Map Id Token, + -- The shell type, such as Bash or Ksh + shellType :: Shell, + -- True if shell type was forced via flags + shellTypeSpecified :: Bool, + -- The root node of the AST + rootNode :: Token, + -- map from token id to start and end position + tokenPositions :: Map.Map Id (Position, Position) } deriving (Show) -- TODO: Cache results of common AST ops here @@ -154,14 +162,11 @@ warn id code str = addComment $ makeComment WarningC id code str err id code str = addComment $ makeComment ErrorC id code str info id code str = addComment $ makeComment InfoC id code str style id code str = addComment $ makeComment StyleC id code str -verbose id code str = addComment $ makeComment VerboseC id code str warnWithFix :: MonadWriter [TokenComment] m => Id -> Code -> String -> Fix -> m () warnWithFix = addCommentWithFix WarningC styleWithFix :: MonadWriter [TokenComment] m => Id -> Code -> String -> Fix -> m () styleWithFix = addCommentWithFix StyleC -verboseWithFix :: MonadWriter [TokenComment] m => Id -> Code -> String -> Fix -> m () -verboseWithFix = addCommentWithFix VerboseC addCommentWithFix :: MonadWriter [TokenComment] m => Severity -> Id -> Code -> String -> Fix -> m () addCommentWithFix severity id code str fix = diff --git a/src/ShellCheck/Checker.hs b/src/ShellCheck/Checker.hs index e73636d..a231242 100644 --- a/src/ShellCheck/Checker.hs +++ b/src/ShellCheck/Checker.hs @@ -84,7 +84,8 @@ checkScript sys spec = do asFallbackShell = shellFromFilename $ csFilename spec, asCheckSourced = csCheckSourced spec, asExecutionMode = Executed, - asTokenPositions = tokenPositions + asTokenPositions = tokenPositions, + asOptionalChecks = csOptionalChecks spec } where as = newAnalysisSpec root let analysisMessages = fromMaybe [] $ @@ -302,6 +303,14 @@ prop_sourcedFileUsesOriginalShellExtension = result == [2079] csCheckSourced = True } +prop_canEnableOptionalsWithSpec = result == [2244] + where + result = checkWithSpec [] emptyCheckSpec { + csFilename = "file.sh", + csScript = "#!/bin/sh\n[ \"$1\" ]", + csOptionalChecks = ["avoid-nullary-conditions"] + } + prop_optionIncludes1 = -- expect 2086, but not included, so nothing reported null $ checkOptionIncludes (Just [2080]) "#!/bin/sh\n var='a b'\n echo $var" @@ -347,6 +356,12 @@ prop_brokenRcGetsWarning = result == [1134, 2086] csIgnoreRC = False } +prop_canEnableOptionalsWithRc = result == [2244] + where + result = checkWithRc "enable=avoid-nullary-conditions" emptyCheckSpec { + csScript = "#!/bin/sh\n[ \"$1\" ]" + } + prop_sourcePathRedirectsName = result == [2086] where f "dir/myscript" _ "lib" = return "foo/lib" diff --git a/src/ShellCheck/Formatter/Format.hs b/src/ShellCheck/Formatter/Format.hs index bb513cd..57b9d71 100644 --- a/src/ShellCheck/Formatter/Format.hs +++ b/src/ShellCheck/Formatter/Format.hs @@ -47,7 +47,6 @@ severityText pc = WarningC -> "warning" InfoC -> "info" StyleC -> "style" - VerboseC -> "verbose" -- Realign comments from a tabstop of 8 to 1 makeNonVirtual comments contents = diff --git a/src/ShellCheck/Interface.hs b/src/ShellCheck/Interface.hs index 1d0cc6b..e60433e 100644 --- a/src/ShellCheck/Interface.hs +++ b/src/ShellCheck/Interface.hs @@ -21,18 +21,18 @@ module ShellCheck.Interface ( SystemInterface(..) - , CheckSpec(csFilename, csScript, csCheckSourced, csIncludedWarnings, csExcludedWarnings, csShellTypeOverride, csMinSeverity, csIgnoreRC) + , CheckSpec(csFilename, csScript, csCheckSourced, csIncludedWarnings, csExcludedWarnings, csShellTypeOverride, csMinSeverity, csIgnoreRC, csOptionalChecks) , CheckResult(crFilename, crComments) , ParseSpec(psFilename, psScript, psCheckSourced, psIgnoreRC, psShellTypeOverride) , ParseResult(prComments, prTokenPositions, prRoot) - , AnalysisSpec(asScript, asShellType, asFallbackShell, asExecutionMode, asCheckSourced, asTokenPositions) + , AnalysisSpec(asScript, asShellType, asFallbackShell, asExecutionMode, asCheckSourced, asTokenPositions, asOptionalChecks) , AnalysisResult(arComments) , FormatterOptions(foColorOption, foWikiLinkCount) , Shell(Ksh, Sh, Bash, Dash) , ExecutionMode(Executed, Sourced) , ErrorMessage , Code - , Severity(ErrorC, WarningC, InfoC, StyleC, VerboseC) + , Severity(ErrorC, WarningC, InfoC, StyleC) , Position(posFile, posLine, posColumn) , Comment(cSeverity, cCode, cMessage) , PositionedComment(pcStartPos , pcEndPos , pcComment, pcFix) @@ -56,6 +56,8 @@ module ShellCheck.Interface , InsertionPoint(InsertBefore, InsertAfter) , Replacement(repStartPos, repEndPos, repString, repPrecedence, repInsertionPoint) , newReplacement + , CheckDescription(cdName, cdDescription, cdPositive, cdNegative) + , newCheckDescription ) where import ShellCheck.AST @@ -92,7 +94,8 @@ data CheckSpec = CheckSpec { csExcludedWarnings :: [Integer], csIncludedWarnings :: Maybe [Integer], csShellTypeOverride :: Maybe Shell, - csMinSeverity :: Severity + csMinSeverity :: Severity, + csOptionalChecks :: [String] } deriving (Show, Eq) data CheckResult = CheckResult { @@ -115,7 +118,8 @@ emptyCheckSpec = CheckSpec { csExcludedWarnings = [], csIncludedWarnings = Nothing, csShellTypeOverride = Nothing, - csMinSeverity = StyleC + csMinSeverity = StyleC, + csOptionalChecks = [] } newParseSpec :: ParseSpec @@ -156,6 +160,7 @@ data AnalysisSpec = AnalysisSpec { asFallbackShell :: Maybe Shell, asExecutionMode :: ExecutionMode, asCheckSourced :: Bool, + asOptionalChecks :: [String], asTokenPositions :: Map.Map Id (Position, Position) } @@ -165,6 +170,7 @@ newAnalysisSpec token = AnalysisSpec { asFallbackShell = Nothing, asExecutionMode = Executed, asCheckSourced = False, + asOptionalChecks = [], asTokenPositions = Map.empty } @@ -187,6 +193,19 @@ newFormatterOptions = FormatterOptions { foWikiLinkCount = 3 } +data CheckDescription = CheckDescription { + cdName :: String, + cdDescription :: String, + cdPositive :: String, + cdNegative :: String + } + +newCheckDescription = CheckDescription { + cdName = "", + cdDescription = "", + cdPositive = "", + cdNegative = "" + } -- Supporting data types data Shell = Ksh | Sh | Bash | Dash deriving (Show, Eq) @@ -195,7 +214,7 @@ data ExecutionMode = Executed | Sourced deriving (Show, Eq) type ErrorMessage = String type Code = Integer -data Severity = ErrorC | WarningC | InfoC | StyleC | VerboseC +data Severity = ErrorC | WarningC | InfoC | StyleC deriving (Show, Eq, Ord, Generic, NFData) data Position = Position { posFile :: String, -- Filename diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index f5a52ed..a403bea 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -985,6 +985,10 @@ readAnnotationWithoutPrefix = do int <- many1 digit return $ DisableComment (read int) + "enable" -> readName `sepBy` char ',' + where + readName = EnableComment <$> many1 (letter <|> char '-') + "source" -> do filename <- many1 $ noneOf " \n" return [SourceOverride filename] From 50af8aba29199ae3e008e38de1bc9463ca1777f8 Mon Sep 17 00:00:00 2001 From: Benjamin Gordon Date: Tue, 7 May 2019 15:49:34 -0600 Subject: [PATCH 126/763] Add json1 format that ignores tabs The new json1 format works just like json except that it treats tabs as single characters instead of 8-character tabstops. The main use case is to allow editors to pass -fjson1 so that they can consume the json output in a character-oriented way without breaking backwards compatibility. Also addresses #1048. --- CHANGELOG.md | 1 + shellcheck.1.md | 7 ++++++- shellcheck.hs | 3 ++- src/ShellCheck/Formatter/Format.hs | 8 +++++++- src/ShellCheck/Formatter/JSON.hs | 21 +++++++++++++++++---- 5 files changed, 33 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bb623f0..797bf67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ Enable with `-o` flags or `enable=name` directives - Source paths: Use `-P dir1:dir2` or a `source-path=dir1` directive to specify search paths for sourced files. +- json1 format like --format=json but treats tabs as single characters - SC2249: Warn about `case` with missing default case (verbose) - SC2248: Warn about unquoted variables without special chars (verbose) - SC2247: Warn about $"(cmd)" and $"{var}" diff --git a/shellcheck.1.md b/shellcheck.1.md index f8cbd44..e99e8ea 100644 --- a/shellcheck.1.md +++ b/shellcheck.1.md @@ -153,7 +153,7 @@ not warn at all, as `ksh` supports decimals in arithmetic contexts. : Json is a popular serialization format that is more suitable for web applications. ShellCheck's json is compact and contains only the bare - minimum. + minimum. Tabs are 8 characters. [ { @@ -167,6 +167,11 @@ not warn at all, as `ksh` supports decimals in arithmetic contexts. ... ] +**json1** + +: This is the same as shellcheck's json format, but tabs are treated as + single characters instead of 8-character tabstops. + *quiet* : Suppress all normal output. Exit with zero if no issues are found, diff --git a/shellcheck.hs b/shellcheck.hs index c8ea7fb..cc5050e 100644 --- a/shellcheck.hs +++ b/shellcheck.hs @@ -141,7 +141,8 @@ formats :: FormatterOptions -> Map.Map String (IO Formatter) formats options = Map.fromList [ ("checkstyle", ShellCheck.Formatter.CheckStyle.format), ("gcc", ShellCheck.Formatter.GCC.format), - ("json", ShellCheck.Formatter.JSON.format), + ("json", ShellCheck.Formatter.JSON.format False), -- JSON with 8-char tabs + ("json1", ShellCheck.Formatter.JSON.format True), -- JSON with 1-char tabs ("tty", ShellCheck.Formatter.TTY.format options), ("quiet", ShellCheck.Formatter.Quiet.format options) ] diff --git a/src/ShellCheck/Formatter/Format.hs b/src/ShellCheck/Formatter/Format.hs index 57b9d71..4a3908a 100644 --- a/src/ShellCheck/Formatter/Format.hs +++ b/src/ShellCheck/Formatter/Format.hs @@ -22,6 +22,7 @@ module ShellCheck.Formatter.Format where import ShellCheck.Data import ShellCheck.Interface import ShellCheck.Fixer +import Control.Monad import Data.Array -- A formatter that carries along an arbitrary piece of data @@ -54,5 +55,10 @@ makeNonVirtual comments contents = where list = lines contents arr = listArray (1, length list) list - fix c = removeTabStops c arr + untabbedFix f = newFix { + fixReplacements = map (\r -> removeTabStops r arr) (fixReplacements f) + } + fix c = (removeTabStops c arr) { + pcFix = liftM untabbedFix (pcFix c) + } diff --git a/src/ShellCheck/Formatter/JSON.hs b/src/ShellCheck/Formatter/JSON.hs index 02a549d..72f90fa 100644 --- a/src/ShellCheck/Formatter/JSON.hs +++ b/src/ShellCheck/Formatter/JSON.hs @@ -30,11 +30,12 @@ import GHC.Exts import System.IO import qualified Data.ByteString.Lazy.Char8 as BL -format = do +format :: Bool -> IO Formatter +format removeTabs = do ref <- newIORef [] return Formatter { header = return (), - onResult = collectResult ref, + onResult = collectResult removeTabs ref, onFailure = outputError, footer = finish ref } @@ -96,8 +97,20 @@ instance ToJSON Fix where ] outputError file msg = hPutStrLn stderr $ file ++ ": " ++ msg -collectResult ref result _ = - modifyIORef ref (\x -> crComments result ++ x) + +collectResult removeTabs ref cr sys = mapM_ f groups + where + comments = crComments cr + groups = groupWith sourceFile comments + f :: [PositionedComment] -> IO () + f group = do + let filename = sourceFile (head group) + result <- siReadFile sys filename + let contents = either (const "") id result + let comments' = if removeTabs + then makeNonVirtual comments contents + else comments + modifyIORef ref (\x -> comments' ++ x) finish ref = do list <- readIORef ref From 5ccaddbcc2455c7b5c753014b16dcb9f8ec78bdb Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 13 May 2019 19:31:23 -0700 Subject: [PATCH 127/763] Promote `json1` as the primary JSON format --- shellcheck.1.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/shellcheck.1.md b/shellcheck.1.md index e99e8ea..c0e3817 100644 --- a/shellcheck.1.md +++ b/shellcheck.1.md @@ -149,11 +149,11 @@ not warn at all, as `ksh` supports decimals in arithmetic contexts. ... -**json** +**json1** : Json is a popular serialization format that is more suitable for web applications. ShellCheck's json is compact and contains only the bare - minimum. Tabs are 8 characters. + minimum. Tabs are counted as 1 character. [ { @@ -167,12 +167,12 @@ not warn at all, as `ksh` supports decimals in arithmetic contexts. ... ] -**json1** +**json** -: This is the same as shellcheck's json format, but tabs are treated as - single characters instead of 8-character tabstops. +: This is a legacy version of the **json1** format, with a tab stop + of 8 instead of 1. -*quiet* +**quiet** : Suppress all normal output. Exit with zero if no issues are found, otherwise exit with one. Stops processing after the first issue. From 50116e8aee35988e7dbb3fdd67bf62edf083940a Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 13 May 2019 20:45:53 -0700 Subject: [PATCH 128/763] Don't suggest [[..]] for sh in SC2081 (fixes #1562) --- src/ShellCheck/Analytics.hs | 10 ++++++++-- src/ShellCheck/AnalyzerLib.hs | 9 +++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index ac8fcdb..c64205f 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1302,12 +1302,18 @@ prop_checkComparisonAgainstGlob3 = verify checkComparisonAgainstGlob "[ $cow = * prop_checkComparisonAgainstGlob4 = verifyNot checkComparisonAgainstGlob "[ $cow = foo ]" prop_checkComparisonAgainstGlob5 = verify checkComparisonAgainstGlob "[[ $cow != $bar ]]" prop_checkComparisonAgainstGlob6 = verify checkComparisonAgainstGlob "[ $f != /* ]" + checkComparisonAgainstGlob _ (TC_Binary _ DoubleBracket op _ (T_NormalWord id [T_DollarBraced _ _])) | op `elem` ["=", "==", "!="] = warn id 2053 $ "Quote the right-hand side of " ++ op ++ " in [[ ]] to prevent glob matching." -checkComparisonAgainstGlob _ (TC_Binary _ SingleBracket op _ word) +checkComparisonAgainstGlob params (TC_Binary _ SingleBracket op _ word) | op `elem` ["=", "==", "!="] && isGlob word = - err (getId word) 2081 "[ .. ] can't match globs. Use [[ .. ]] or case statement." + err (getId word) 2081 msg + where + msg = if isBashLike params + then "[ .. ] can't match globs. Use [[ .. ]] or case statement." + else "[ .. ] can't match globs. Use a case statement." + checkComparisonAgainstGlob _ _ = return () prop_checkCommarrays1 = verify checkCommarrays "a=(1, 2)" diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 5783820..8185565 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -947,5 +947,14 @@ getOpts flagTokenizer string cmd = process flags supportsArrays shell = shell == Bash || shell == Ksh +-- Returns true if the shell is Bash or Ksh (sorry for the name, Ksh) +isBashLike :: Parameters -> Bool +isBashLike params = + case shellType params of + Bash -> True + Ksh -> True + Dash -> False + Sh -> False + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) From ea05271fa33966c749bb516dd65365b36ba1a88c Mon Sep 17 00:00:00 2001 From: Virgil Date: Tue, 14 May 2019 02:32:02 +1000 Subject: [PATCH 129/763] :memo: Update Copyright to year 2019 and Markdown linting - [x] :memo: Update Copyright to year 2019 - [x] :pencil: MD009/no-trailing-spaces: Trailing spaces [Expected: 0 or 2; Actual: 1] - [x] :pencil: MD034/no-bare-urls: Bare URL used - [ ] :pencil: ~MD004/ul-style: Unordered list style [Expected: dash; Actual: asterisk]~ - [ ] ~add missing TOC entries~ --- README.md | 67 +++++++++++++------------- shellcheck.1.md | 2 +- shellcheck.hs | 2 +- src/ShellCheck/AST.hs | 2 +- src/ShellCheck/ASTLib.hs | 2 +- src/ShellCheck/Analytics.hs | 2 +- src/ShellCheck/Analyzer.hs | 2 +- src/ShellCheck/AnalyzerLib.hs | 2 +- src/ShellCheck/Checker.hs | 2 +- src/ShellCheck/Checks/Commands.hs | 2 +- src/ShellCheck/Formatter/CheckStyle.hs | 2 +- src/ShellCheck/Formatter/Format.hs | 2 +- src/ShellCheck/Formatter/GCC.hs | 2 +- src/ShellCheck/Formatter/JSON.hs | 2 +- src/ShellCheck/Formatter/TTY.hs | 2 +- src/ShellCheck/Interface.hs | 2 +- src/ShellCheck/Parser.hs | 2 +- src/ShellCheck/Regex.hs | 2 +- 18 files changed, 50 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index a88aa31..541c790 100644 --- a/README.md +++ b/README.md @@ -8,46 +8,45 @@ ShellCheck is a GPLv3 tool that gives warnings and suggestions for bash/sh shell The goals of ShellCheck are -- To point out and clarify typical beginner's syntax issues that cause a shell +* To point out and clarify typical beginner's syntax issues that cause a shell to give cryptic error messages. -- To point out and clarify typical intermediate level semantic problems that +* To point out and clarify typical intermediate level semantic problems that cause a shell to behave strangely and counter-intuitively. -- To point out subtle caveats, corner cases and pitfalls that may cause an +* To point out subtle caveats, corner cases and pitfalls that may cause an advanced user's otherwise working script to fail under future circumstances. See [the gallery of bad code](README.md#user-content-gallery-of-bad-code) for examples of what ShellCheck can help you identify! ## Table of Contents -- [Table of Contents](#table-of-contents) -- [How to use](#how-to-use) - - [On the web](#on-the-web) - - [From your terminal](#from-your-terminal) - - [In your editor](#in-your-editor) - - [In your build or test suites](#in-your-build-or-test-suites) -- [Installing](#installing) -- [Compiling from source](#compiling-from-source) - - [Installing Cabal](#installing-cabal) - - [Compiling ShellCheck](#compiling-shellcheck) - - [Running tests](#running-tests) -- [Gallery of bad code](#gallery-of-bad-code) - - [Quoting](#quoting) - - [Conditionals](#conditionals) - - [Frequently misused commands](#frequently-misused-commands) - - [Common beginner's mistakes](#common-beginners-mistakes) - - [Style](#style) - - [Data and typing errors](#data-and-typing-errors) - - [Robustness](#robustness) - - [Portability](#portability) - - [Miscellaneous](#miscellaneous) -- [Testimonials](#testimonials) -- [Ignoring issues](#ignoring-issues) -- [Reporting bugs](#reporting-bugs) -- [Contributing](#contributing) -- [Copyright](#copyright) -- [Other Resources](#other-resources) +* [How to use](#how-to-use) + * [On the web](#on-the-web) + * [From your terminal](#from-your-terminal) + * [In your editor](#in-your-editor) + * [In your build or test suites](#in-your-build-or-test-suites) +* [Installing](#installing) +* [Compiling from source](#compiling-from-source) + * [Installing Cabal](#installing-cabal) + * [Compiling ShellCheck](#compiling-shellcheck) + * [Running tests](#running-tests) +* [Gallery of bad code](#gallery-of-bad-code) + * [Quoting](#quoting) + * [Conditionals](#conditionals) + * [Frequently misused commands](#frequently-misused-commands) + * [Common beginner's mistakes](#common-beginners-mistakes) + * [Style](#style) + * [Data and typing errors](#data-and-typing-errors) + * [Robustness](#robustness) + * [Portability](#portability) + * [Miscellaneous](#miscellaneous) +* [Testimonials](#testimonials) +* [Ignoring issues](#ignoring-issues) +* [Reporting bugs](#reporting-bugs) +* [Contributing](#contributing) +* [Copyright](#copyright) +* [Other Resources](#other-resources) ## How to use @@ -101,7 +100,7 @@ or in a Travis CI `.travis.yml` file: ```yaml script: # Fail if any of these files have warnings - - shellcheck myscripts/*.sh + * shellcheck myscripts/*.sh ``` Services and platforms that have ShellCheck pre-installed and ready to use: @@ -179,7 +178,7 @@ On openSUSE zypper in ShellCheck -Or use OneClickInstall - https://software.opensuse.org/package/ShellCheck +Or use OneClickInstall - On Solus: @@ -262,7 +261,7 @@ On MacOS (OS X), you can do a fast install of Cabal using brew, which takes a co brew cask install haskell-platform cabal install cabal-install -On MacPorts, the package is instead called `hs-cabal-install`, while native Windows users should install the latest version of the Haskell platform from https://www.haskell.org/platform/ +On MacPorts, the package is instead called `hs-cabal-install`, while native Windows users should install the latest version of the Haskell platform from Verify that `cabal` is installed and update its dependency list with @@ -505,7 +504,7 @@ The contributor retains the copyright. ShellCheck is licensed under the GNU General Public License, v3. A copy of this license is included in the file [LICENSE](LICENSE). -Copyright 2012-2018, Vidar 'koala_man' Holen and contributors. +Copyright 2012-2019, [Vidar 'koala_man' Holen](https://github.com/koalaman/) and contributors. Happy ShellChecking! diff --git a/shellcheck.1.md b/shellcheck.1.md index f8cbd44..1612526 100644 --- a/shellcheck.1.md +++ b/shellcheck.1.md @@ -294,7 +294,7 @@ Bugs and issues can be reported on GitHub: https://github.com/koalaman/shellcheck/issues # COPYRIGHT -Copyright 2012-2015, Vidar Holen. +Copyright 2012-2019, Vidar Holen. Licensed under the GNU General Public License version 3 or later, see https://gnu.org/licenses/gpl.html diff --git a/shellcheck.hs b/shellcheck.hs index c8ea7fb..bef4101 100644 --- a/shellcheck.hs +++ b/shellcheck.hs @@ -1,5 +1,5 @@ {- - Copyright 2012-2015 Vidar Holen + Copyright 2012-2019 Vidar Holen This file is part of ShellCheck. https://www.shellcheck.net diff --git a/src/ShellCheck/AST.hs b/src/ShellCheck/AST.hs index 5b4254f..1459ca6 100644 --- a/src/ShellCheck/AST.hs +++ b/src/ShellCheck/AST.hs @@ -1,5 +1,5 @@ {- - Copyright 2012-2015 Vidar Holen + Copyright 2012-2019 Vidar Holen This file is part of ShellCheck. https://www.shellcheck.net diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 7e8023e..d264d02 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -1,5 +1,5 @@ {- - Copyright 2012-2015 Vidar Holen + Copyright 2012-2019 Vidar Holen This file is part of ShellCheck. https://www.shellcheck.net diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index ac8fcdb..7926b53 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1,5 +1,5 @@ {- - Copyright 2012-2015 Vidar Holen + Copyright 2012-2019 Vidar Holen This file is part of ShellCheck. https://www.shellcheck.net diff --git a/src/ShellCheck/Analyzer.hs b/src/ShellCheck/Analyzer.hs index 442daba..01440d8 100644 --- a/src/ShellCheck/Analyzer.hs +++ b/src/ShellCheck/Analyzer.hs @@ -1,5 +1,5 @@ {- - Copyright 2012-2015 Vidar Holen + Copyright 2012-2019 Vidar Holen This file is part of ShellCheck. https://www.shellcheck.net diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 5783820..6142475 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -1,5 +1,5 @@ {- - Copyright 2012-2015 Vidar Holen + Copyright 2012-2019 Vidar Holen This file is part of ShellCheck. https://www.shellcheck.net diff --git a/src/ShellCheck/Checker.hs b/src/ShellCheck/Checker.hs index a231242..2ea950d 100644 --- a/src/ShellCheck/Checker.hs +++ b/src/ShellCheck/Checker.hs @@ -1,5 +1,5 @@ {- - Copyright 2012-2015 Vidar Holen + Copyright 2012-2019 Vidar Holen This file is part of ShellCheck. https://www.shellcheck.net diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 346d880..f9f38ea 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -1,5 +1,5 @@ {- - Copyright 2012-2015 Vidar Holen + Copyright 2012-2019 Vidar Holen This file is part of ShellCheck. https://www.shellcheck.net diff --git a/src/ShellCheck/Formatter/CheckStyle.hs b/src/ShellCheck/Formatter/CheckStyle.hs index b3f2074..f3fea88 100644 --- a/src/ShellCheck/Formatter/CheckStyle.hs +++ b/src/ShellCheck/Formatter/CheckStyle.hs @@ -1,5 +1,5 @@ {- - Copyright 2012-2015 Vidar Holen + Copyright 2012-2019 Vidar Holen This file is part of ShellCheck. https://www.shellcheck.net diff --git a/src/ShellCheck/Formatter/Format.hs b/src/ShellCheck/Formatter/Format.hs index 57b9d71..61c5025 100644 --- a/src/ShellCheck/Formatter/Format.hs +++ b/src/ShellCheck/Formatter/Format.hs @@ -1,5 +1,5 @@ {- - Copyright 2012-2015 Vidar Holen + Copyright 2012-2019 Vidar Holen This file is part of ShellCheck. https://www.shellcheck.net diff --git a/src/ShellCheck/Formatter/GCC.hs b/src/ShellCheck/Formatter/GCC.hs index b8a0bb0..9c5fa5f 100644 --- a/src/ShellCheck/Formatter/GCC.hs +++ b/src/ShellCheck/Formatter/GCC.hs @@ -1,5 +1,5 @@ {- - Copyright 2012-2015 Vidar Holen + Copyright 2012-2019 Vidar Holen This file is part of ShellCheck. https://www.shellcheck.net diff --git a/src/ShellCheck/Formatter/JSON.hs b/src/ShellCheck/Formatter/JSON.hs index 02a549d..0c3185b 100644 --- a/src/ShellCheck/Formatter/JSON.hs +++ b/src/ShellCheck/Formatter/JSON.hs @@ -1,6 +1,6 @@ {-# LANGUAGE OverloadedStrings #-} {- - Copyright 2012-2015 Vidar Holen + Copyright 2012-2019 Vidar Holen This file is part of ShellCheck. https://www.shellcheck.net diff --git a/src/ShellCheck/Formatter/TTY.hs b/src/ShellCheck/Formatter/TTY.hs index c0d5841..845feeb 100644 --- a/src/ShellCheck/Formatter/TTY.hs +++ b/src/ShellCheck/Formatter/TTY.hs @@ -1,5 +1,5 @@ {- - Copyright 2012-2015 Vidar Holen + Copyright 2012-2019 Vidar Holen This file is part of ShellCheck. https://www.shellcheck.net diff --git a/src/ShellCheck/Interface.hs b/src/ShellCheck/Interface.hs index e60433e..aa12fc2 100644 --- a/src/ShellCheck/Interface.hs +++ b/src/ShellCheck/Interface.hs @@ -1,5 +1,5 @@ {- - Copyright 2012-2015 Vidar Holen + Copyright 2012-2019 Vidar Holen This file is part of ShellCheck. https://www.shellcheck.net diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index a403bea..5f41c90 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -1,5 +1,5 @@ {- - Copyright 2012-2015 Vidar Holen + Copyright 2012-2019 Vidar Holen This file is part of ShellCheck. https://www.shellcheck.net diff --git a/src/ShellCheck/Regex.hs b/src/ShellCheck/Regex.hs index f1262b4..9367ee7 100644 --- a/src/ShellCheck/Regex.hs +++ b/src/ShellCheck/Regex.hs @@ -1,5 +1,5 @@ {- - Copyright 2012-2015 Vidar Holen + Copyright 2012-2019 Vidar Holen This file is part of ShellCheck. https://www.shellcheck.net From 0358090b3c06bebe0561468c025e768defaa63f7 Mon Sep 17 00:00:00 2001 From: Benjamin Gordon Date: Thu, 25 Apr 2019 09:32:20 -0600 Subject: [PATCH 130/763] Refactor definition of special variables. This ensures that the parser and other places that refer to special variables can use the same list. --- src/ShellCheck/Data.hs | 2 ++ src/ShellCheck/Parser.hs | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Data.hs b/src/ShellCheck/Data.hs index cae07d3..1a994a3 100644 --- a/src/ShellCheck/Data.hs +++ b/src/ShellCheck/Data.hs @@ -47,6 +47,8 @@ variablesWithoutSpaces = specialVariablesWithoutSpaces ++ [ "COLUMNS", "HISTFILESIZE", "HISTSIZE", "LINES" ] +specialVariables = specialVariablesWithoutSpaces ++ ["@", "*"] + arrayVariables = [ "BASH_ALIASES", "BASH_ARGC", "BASH_ARGV", "BASH_CMDS", "BASH_LINENO", "BASH_REMATCH", "BASH_SOURCE", "BASH_VERSINFO", "COMP_WORDS", "COPROC", diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index a403bea..f5f62cd 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -69,7 +69,7 @@ variableChars = upper <|> lower <|> digit <|> oneOf "_" functionChars = variableChars <|> oneOf ":+?-./^@" -- Chars to allow in functions using the 'function' keyword extendedFunctionChars = functionChars <|> oneOf "[]*=!" -specialVariable = oneOf "@*#?-$!" +specialVariable = oneOf (concat specialVariables) paramSubSpecialChars = oneOf "/:+-=%" quotableChars = "|&;<>()\\ '\t\n\r\xA0" ++ doubleQuotableChars quotable = almostSpace <|> oneOf quotableChars From aa3b709b5d7011e480203f8620c9ff8a64c2d0e3 Mon Sep 17 00:00:00 2001 From: Benjamin Gordon Date: Mon, 29 Apr 2019 14:26:39 -0600 Subject: [PATCH 131/763] Track whether braces were present in T_DollarBraced References of the form $var and ${var} both map to the same structure in the AST, which prevents any later analysis functions from distinguishing them. In preparation for adding checks that need this info, add a Bool to T_DollarBraced that tracks whether the braces were seen at parsing time and update all references so that this change is a no-op. --- src/ShellCheck/AST.hs | 6 ++--- src/ShellCheck/ASTLib.hs | 8 +++---- src/ShellCheck/Analytics.hs | 33 +++++++++++++-------------- src/ShellCheck/AnalyzerLib.hs | 8 +++---- src/ShellCheck/Checks/Commands.hs | 4 ++-- src/ShellCheck/Checks/ShellSupport.hs | 2 +- src/ShellCheck/Parser.hs | 6 ++--- 7 files changed, 33 insertions(+), 34 deletions(-) diff --git a/src/ShellCheck/AST.hs b/src/ShellCheck/AST.hs index 5b4254f..86c04ee 100644 --- a/src/ShellCheck/AST.hs +++ b/src/ShellCheck/AST.hs @@ -76,7 +76,7 @@ data Token = | T_DSEMI Id | T_Do Id | T_DollarArithmetic Id Token - | T_DollarBraced Id Token + | T_DollarBraced Id Bool Token | T_DollarBracket Id Token | T_DollarDoubleQuoted Id [Token] | T_DollarExpansion Id [Token] @@ -253,7 +253,7 @@ analyze f g i = delve (T_Function id a b name body) = d1 body $ T_Function id a b name delve (T_Condition id typ token) = d1 token $ T_Condition id typ delve (T_Extglob id str l) = dl l $ T_Extglob id str - delve (T_DollarBraced id op) = d1 op $ T_DollarBraced id + delve (T_DollarBraced id braced op) = d1 op $ T_DollarBraced id braced delve (T_HereDoc id d q str l) = dl l $ T_HereDoc id d q str delve (TC_And id typ str t1 t2) = d2 t1 t2 $ TC_And id typ str @@ -323,7 +323,7 @@ getId t = case t of T_NormalWord id _ -> id T_DoubleQuoted id _ -> id T_DollarExpansion id _ -> id - T_DollarBraced id _ -> id + T_DollarBraced id _ _ -> id T_DollarArithmetic id _ -> id T_BraceExpansion id _ -> id T_ParamSubSpecialChar id _ -> id diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 7e8023e..a8970e9 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -81,7 +81,7 @@ oversimplify token = (T_NormalWord _ l) -> [concat (concatMap oversimplify l)] (T_DoubleQuoted _ l) -> [concat (concatMap oversimplify l)] (T_SingleQuoted _ s) -> [s] - (T_DollarBraced _ _) -> ["${VAR}"] + (T_DollarBraced _ _ _) -> ["${VAR}"] (T_DollarArithmetic _ _) -> ["${VAR}"] (T_DollarExpansion _ _) -> ["${VAR}"] (T_Backticked _ _) -> ["${VAR}"] @@ -133,11 +133,11 @@ isUnquotedFlag token = fromMaybe False $ do return $ "-" `isPrefixOf` str -- Given a T_DollarBraced, return a simplified version of the string contents. -bracedString (T_DollarBraced _ l) = concat $ oversimplify l +bracedString (T_DollarBraced _ _ l) = concat $ oversimplify l bracedString _ = error "Internal shellcheck error, please report! (bracedString on non-variable)" -- Is this an expansion of multiple items of an array? -isArrayExpansion t@(T_DollarBraced _ _) = +isArrayExpansion t@(T_DollarBraced _ _ _) = let string = bracedString t in "@" `isPrefixOf` string || not ("#" `isPrefixOf` string) && "[@]" `isInfixOf` string @@ -146,7 +146,7 @@ isArrayExpansion _ = False -- Is it possible that this arg becomes multiple args? mayBecomeMultipleArgs t = willBecomeMultipleArgs t || f t where - f t@(T_DollarBraced _ _) = + f t@(T_DollarBraced _ _ _) = let string = bracedString t in "!" `isPrefixOf` string f (T_DoubleQuoted _ parts) = any f parts diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index c64205f..9ddc2d9 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -754,7 +754,7 @@ checkShorthandIf _ _ = return () prop_checkDollarStar = verify checkDollarStar "for f in $*; do ..; done" prop_checkDollarStar2 = verifyNot checkDollarStar "a=$*" prop_checkDollarStar3 = verifyNot checkDollarStar "[[ $* = 'a b' ]]" -checkDollarStar p t@(T_NormalWord _ [b@(T_DollarBraced id _)]) +checkDollarStar p t@(T_NormalWord _ [b@(T_DollarBraced id _ _)]) | bracedString b == "*" = unless (isStrictlyQuoteFree (parentMap p) t) $ warn id 2048 "Use \"$@\" (with quotes) to prevent whitespace problems." @@ -825,7 +825,7 @@ checkArrayWithoutIndex params _ = doVariableFlowAnalysis readF writeF defaultMap (variableFlow params) where defaultMap = Map.fromList $ map (\x -> (x,())) arrayVariables - readF _ (T_DollarBraced id token) _ = do + readF _ (T_DollarBraced id _ token) _ = do map <- get return . maybeToList $ do name <- getLiteralString token @@ -1267,7 +1267,7 @@ prop_checkArithmeticDeref12= verify checkArithmeticDeref "for ((i=0; $i < 3; i)) prop_checkArithmeticDeref13= verifyNot checkArithmeticDeref "(( $$ ))" prop_checkArithmeticDeref14= verifyNot checkArithmeticDeref "(( $! ))" prop_checkArithmeticDeref15= verifyNot checkArithmeticDeref "(( ${!var} ))" -checkArithmeticDeref params t@(TA_Expansion _ [b@(T_DollarBraced id _)]) = +checkArithmeticDeref params t@(TA_Expansion _ [b@(T_DollarBraced id _ _)]) = unless (isException $ bracedString b) getWarning where isException [] = True @@ -1302,8 +1302,7 @@ prop_checkComparisonAgainstGlob3 = verify checkComparisonAgainstGlob "[ $cow = * prop_checkComparisonAgainstGlob4 = verifyNot checkComparisonAgainstGlob "[ $cow = foo ]" prop_checkComparisonAgainstGlob5 = verify checkComparisonAgainstGlob "[[ $cow != $bar ]]" prop_checkComparisonAgainstGlob6 = verify checkComparisonAgainstGlob "[ $f != /* ]" - -checkComparisonAgainstGlob _ (TC_Binary _ DoubleBracket op _ (T_NormalWord id [T_DollarBraced _ _])) +checkComparisonAgainstGlob _ (TC_Binary _ DoubleBracket op _ (T_NormalWord id [T_DollarBraced _ _ _])) | op `elem` ["=", "==", "!="] = warn id 2053 $ "Quote the right-hand side of " ++ op ++ " in [[ ]] to prevent glob matching." checkComparisonAgainstGlob params (TC_Binary _ SingleBracket op _ word) @@ -1457,7 +1456,7 @@ prop_checkIndirectExpansion2 = verifyNot checkIndirectExpansion "${foo//$n/lol}" prop_checkIndirectExpansion3 = verify checkIndirectExpansion "${$#}" prop_checkIndirectExpansion4 = verify checkIndirectExpansion "${var${n}_$((i%2))}" prop_checkIndirectExpansion5 = verifyNot checkIndirectExpansion "${bar}" -checkIndirectExpansion _ (T_DollarBraced i (T_NormalWord _ contents)) = +checkIndirectExpansion _ (T_DollarBraced i _ (T_NormalWord _ contents)) = when (isIndirection contents) $ err i 2082 "To expand via indirection, use arrays, ${!name} or (for sh only) eval." where @@ -1467,7 +1466,7 @@ checkIndirectExpansion _ (T_DollarBraced i (T_NormalWord _ contents)) = isIndirectionPart t = case t of T_DollarExpansion _ _ -> Just True T_Backticked _ _ -> Just True - T_DollarBraced _ _ -> Just True + T_DollarBraced _ _ _ -> Just True T_DollarArithmetic _ _ -> Just True T_Literal _ s -> if all isVariableChar s then Nothing @@ -1494,7 +1493,7 @@ checkInexplicablyUnquoted params (T_NormalWord id tokens) = mapM_ check (tails t check (T_DoubleQuoted _ a:trapped:T_DoubleQuoted _ b:_) = case trapped of T_DollarExpansion id _ -> warnAboutExpansion id - T_DollarBraced id _ -> warnAboutExpansion id + T_DollarBraced id _ _ -> warnAboutExpansion id T_Literal id s -> unless (quotesSingleThing a && quotesSingleThing b || isRegex (getPath (parentMap params) trapped)) $ warnAboutLiteral id @@ -1515,7 +1514,7 @@ checkInexplicablyUnquoted params (T_NormalWord id tokens) = mapM_ check (tails t -- the quotes were probably intentional and harmless. quotesSingleThing x = case x of [T_DollarExpansion _ _] -> True - [T_DollarBraced _ _] -> True + [T_DollarBraced _ _ _] -> True [T_Backticked _ _] -> True _ -> False @@ -1822,7 +1821,7 @@ checkSpacefulness' onFind params t = isExpansion t = case t of - (T_DollarBraced _ _ ) -> True + (T_DollarBraced _ _ _ ) -> True _ -> False isSpacefulWord :: (String -> Bool) -> [Token] -> Bool @@ -1836,7 +1835,7 @@ checkSpacefulness' onFind params t = T_Extglob {} -> True T_Literal _ s -> s `containsAny` globspace T_SingleQuoted _ s -> s `containsAny` globspace - T_DollarBraced _ _ -> spacefulF $ getBracedReference $ bracedString x + T_DollarBraced _ _ _ -> spacefulF $ getBracedReference $ bracedString x T_NormalWord _ w -> isSpacefulWord spacefulF w T_DoubleQuoted _ w -> isSpacefulWord spacefulF w _ -> False @@ -1874,7 +1873,7 @@ checkQuotesInLiterals params t = return [] writeF _ _ _ _ = return [] - forToken map (T_DollarBraced id t) = + forToken map (T_DollarBraced id _ t) = -- skip getBracedReference here to avoid false positives on PE Map.lookup (concat . oversimplify $ t) map forToken quoteMap (T_DoubleQuoted id tokens) = @@ -1888,7 +1887,7 @@ checkQuotesInLiterals params t = squashesQuotes t = case t of - T_DollarBraced id _ -> "#" `isPrefixOf` bracedString t + T_DollarBraced id _ _ -> "#" `isPrefixOf` bracedString t _ -> False readF _ expr name = do @@ -2135,10 +2134,10 @@ checkUnassignedReferences params t = warnings isInArray var t = any isArray $ getPath (parentMap params) t where isArray T_Array {} = True - isArray b@(T_DollarBraced _ _) | var /= getBracedReference (bracedString b) = True + isArray b@(T_DollarBraced _ _ _) | var /= getBracedReference (bracedString b) = True isArray _ = False - isGuarded (T_DollarBraced _ v) = + isGuarded (T_DollarBraced _ _ v) = rest `matches` guardRegex where name = concat $ oversimplify v @@ -2224,7 +2223,7 @@ checkWhileReadPitfalls _ _ = return () prop_checkPrefixAssign1 = verify checkPrefixAssignmentReference "var=foo echo $var" prop_checkPrefixAssign2 = verifyNot checkPrefixAssignmentReference "var=$(echo $var) cmd" -checkPrefixAssignmentReference params t@(T_DollarBraced id value) = +checkPrefixAssignmentReference params t@(T_DollarBraced id _ value) = check path where name = getBracedReference $ bracedString t @@ -3026,7 +3025,7 @@ checkSplittingInArrays params t = T_DollarExpansion id _ -> forCommand id T_DollarBraceCommandExpansion id _ -> forCommand id T_Backticked id _ -> forCommand id - T_DollarBraced id str | + T_DollarBraced id _ str | not (isCountingReference part) && not (isQuotedAlternativeReference part) && not (getBracedReference (bracedString part) `elem` variablesWithoutSpaces) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 8185565..e70c940 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -510,7 +510,7 @@ getModifiedVariables t = guard . not . null $ str return (t, token, str, DataString SourceChecked) - T_DollarBraced _ l -> maybeToList $ do + T_DollarBraced _ _ l -> maybeToList $ do let string = bracedString t let modifier = getBracedModifier string guard $ ":=" `isPrefixOf` modifier @@ -702,7 +702,7 @@ getOffsetReferences mods = fromMaybe [] $ do getReferencedVariables parents t = case t of - T_DollarBraced id l -> let str = bracedString t in + T_DollarBraced id _ l -> let str = bracedString t in (t, t, getBracedReference str) : map (\x -> (l, l, x)) ( getIndexReferences str @@ -897,7 +897,7 @@ shouldIgnoreCode params code t = getPath (parentMap params) t -- Is this a ${#anything}, to get string length or array count? -isCountingReference (T_DollarBraced id token) = +isCountingReference (T_DollarBraced id _ token) = case concat $ oversimplify token of '#':_ -> True _ -> False @@ -906,7 +906,7 @@ isCountingReference _ = False -- FIXME: doesn't handle ${a:+$var} vs ${a:+"$var"} isQuotedAlternativeReference t = case t of - T_DollarBraced _ _ -> + T_DollarBraced _ _ _ -> getBracedModifier (bracedString t) `matches` re _ -> False where diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 346d880..fea6932 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -278,7 +278,7 @@ checkTrapQuotes = CommandCheck (Exactly "trap") (f . arguments) where warning id = warn id 2064 "Use single quotes, otherwise this expands now rather than when signalled." checkExpansions (T_DollarExpansion id _) = warning id checkExpansions (T_Backticked id _) = warning id - checkExpansions (T_DollarBraced id _) = warning id + checkExpansions (T_DollarBraced id _ _) = warning id checkExpansions (T_DollarArithmetic id _) = warning id checkExpansions _ = return () @@ -896,7 +896,7 @@ checkCatastrophicRm = CommandCheck (Basename "rm") $ \t -> getPotentialPath = getLiteralStringExt f where f (T_Glob _ str) = return str - f (T_DollarBraced _ word) = + f (T_DollarBraced _ _ word) = let var = onlyLiteralString word in -- This shouldn't handle non-colon cases. if any (`isInfixOf` var) [":?", ":-", ":="] diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 99cdd19..5ca44a1 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -232,7 +232,7 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do bashism t@(TA_Variable id str _) | isBashVariable str = warnMsg id $ str ++ " is" - bashism t@(T_DollarBraced id token) = do + bashism t@(T_DollarBraced id _ token) = do mapM_ check expansion when (isBashVariable var) $ warnMsg id $ var ++ " is" diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index f5f62cd..edbea43 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -1634,7 +1634,7 @@ readDollarBraced = called "parameter expansion" $ do word <- readDollarBracedWord char '}' id <- endSpan start - return $ T_DollarBraced id word + return $ T_DollarBraced id True word prop_readDollarExpansion1= isOk readDollarExpansion "$(echo foo; ls\n)" prop_readDollarExpansion2= isOk readDollarExpansion "$( )" @@ -1661,7 +1661,7 @@ readDollarVariable = do let singleCharred p = do value <- wrapString ((:[]) <$> p) id <- endSpan start - return $ (T_DollarBraced id value) + return $ (T_DollarBraced id False value) let positional = do value <- singleCharred digit @@ -1674,7 +1674,7 @@ readDollarVariable = do let regular = do value <- wrapString readVariableName id <- endSpan start - return (T_DollarBraced id value) `attempting` do + return (T_DollarBraced id False value) `attempting` do lookAhead $ char '[' parseNoteAt pos ErrorC 1087 "Use braces when expanding arrays, e.g. ${array[idx]} (or ${var}[.. to quiet)." From 64c9c83cc8f8ef5dcfb05cf1c756894cd7964c96 Mon Sep 17 00:00:00 2001 From: Benjamin Gordon Date: Tue, 14 May 2019 10:54:56 -0600 Subject: [PATCH 132/763] SC2250: New optional check for braces around variable references Always using braces makes it harder to accidentally change a variable by pasting other text next to it, but the warning is off by default because it's definitely a style preference. Omit special and positional variables from the check because appending additional characters to them already doesn't change parsing. --- CHANGELOG.md | 7 ++++--- src/ShellCheck/Analytics.hs | 25 +++++++++++++++++++++++++ src/ShellCheck/Data.hs | 4 ++++ 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 797bf67..81bd2fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,12 +8,13 @@ - Source paths: Use `-P dir1:dir2` or a `source-path=dir1` directive to specify search paths for sourced files. - json1 format like --format=json but treats tabs as single characters -- SC2249: Warn about `case` with missing default case (verbose) -- SC2248: Warn about unquoted variables without special chars (verbose) +- SC2250: Warn about variable references without braces (optional) +- SC2249: Warn about `case` with missing default case (optional) +- SC2248: Warn about unquoted variables without special chars (optional) - SC2247: Warn about $"(cmd)" and $"{var}" - SC2246: Warn if a shebang's interpreter ends with / - SC2245: Warn that Ksh ignores all but the first glob result in `[` -- SC2243/SC2244: Suggest using explicit -n for `[ $foo ]` (verbose) +- SC2243/SC2244: Suggest using explicit -n for `[ $foo ]` (optional) ### Changed - If a directive or shebang is not specified, a `.bash/.bats/.dash/.ksh` diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 9ddc2d9..edb7cd1 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -223,6 +223,13 @@ optionalTreeChecks = [ cdPositive = "case $? in 0) echo 'Success';; esac", cdNegative = "case $? in 0) echo 'Success';; *) echo 'Fail' ;; esac" }, nodeChecksToTreeCheck [checkDefaultCase]) + + ,(newCheckDescription { + cdName = "require-braces", + cdDescription = "Suggest putting braces around all variable references", + cdPositive = "var=hello; echo $var", + cdNegative = "var=hello; echo ${var}" + }, nodeChecksToTreeCheck [checkVariableBraces]) ] optionalCheckMap :: Map.Map String (Parameters -> Token -> [TokenComment]) @@ -1843,6 +1850,24 @@ checkSpacefulness' onFind params t = globspace = "*?[] \t\n" containsAny s = any (`elem` s) +prop_CheckVariableBraces1 = verify checkVariableBraces "a='123'; echo $a" +prop_CheckVariableBraces2 = verifyNot checkVariableBraces "a='123'; echo ${a}" +prop_CheckVariableBraces3 = verifyNot checkVariableBraces "#shellcheck disable=SC2016\necho '$a'" +prop_CheckVariableBraces4 = verifyNot checkVariableBraces "echo $* $1" +checkVariableBraces params t = + case t of + T_DollarBraced id False _ -> + unless (name `elem` unbracedVariables) $ + styleWithFix id 2250 + "Prefer putting braces around variable references even when not strictly required." + (fixFor t) + + _ -> return () + where + name = getBracedReference $ bracedString t + fixFor token = fixWith [replaceStart (getId token) params 1 "${" + ,replaceEnd (getId token) params 0 "}"] + prop_checkQuotesInLiterals1 = verifyTree checkQuotesInLiterals "param='--foo=\"bar\"'; app $param" prop_checkQuotesInLiterals1a= verifyTree checkQuotesInLiterals "param=\"--foo='lolbar'\"; app $param" prop_checkQuotesInLiterals2 = verifyNotTree checkQuotesInLiterals "param='--foo=\"bar\"'; app \"$param\"" diff --git a/src/ShellCheck/Data.hs b/src/ShellCheck/Data.hs index 1a994a3..2eedeeb 100644 --- a/src/ShellCheck/Data.hs +++ b/src/ShellCheck/Data.hs @@ -49,6 +49,10 @@ variablesWithoutSpaces = specialVariablesWithoutSpaces ++ [ specialVariables = specialVariablesWithoutSpaces ++ ["@", "*"] +unbracedVariables = specialVariables ++ [ + "0", "1", "2", "3", "4", "5", "6", "7", "8", "9" + ] + arrayVariables = [ "BASH_ALIASES", "BASH_ARGC", "BASH_ARGV", "BASH_CMDS", "BASH_LINENO", "BASH_REMATCH", "BASH_SOURCE", "BASH_VERSINFO", "COMP_WORDS", "COPROC", From 861b63aa77e2e7a43a0b9c385c0fa04fdb35dd86 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 14 May 2019 18:48:41 -0700 Subject: [PATCH 133/763] Specify 'variable' in require-braces --- src/ShellCheck/Analytics.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index edb7cd1..d6300d0 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -225,7 +225,7 @@ optionalTreeChecks = [ }, nodeChecksToTreeCheck [checkDefaultCase]) ,(newCheckDescription { - cdName = "require-braces", + cdName = "require-variable-braces", cdDescription = "Suggest putting braces around all variable references", cdPositive = "var=hello; echo $var", cdNegative = "var=hello; echo ${var}" From f5892f2d0d77296d1deebd404204a5014b29d3d8 Mon Sep 17 00:00:00 2001 From: Virgil Date: Fri, 17 May 2019 07:41:12 +1000 Subject: [PATCH 134/763] (docs)Fix typo in yaml markdown Was aligned as per TOC 2nd. Reverted to - --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 541c790..6b789cd 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,7 @@ or in a Travis CI `.travis.yml` file: ```yaml script: # Fail if any of these files have warnings - * shellcheck myscripts/*.sh + - shellcheck myscripts/*.sh ``` Services and platforms that have ShellCheck pre-installed and ready to use: From 8efbecd64aaaa2324e9020bbfd4e4303ff996847 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 19 May 2019 15:29:47 -0700 Subject: [PATCH 135/763] Don't suggest removing braces from $((${x+1})) (fixes #1533) --- src/ShellCheck/Analytics.hs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index e30e899..0db41e8 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1274,11 +1274,12 @@ prop_checkArithmeticDeref12= verify checkArithmeticDeref "for ((i=0; $i < 3; i)) prop_checkArithmeticDeref13= verifyNot checkArithmeticDeref "(( $$ ))" prop_checkArithmeticDeref14= verifyNot checkArithmeticDeref "(( $! ))" prop_checkArithmeticDeref15= verifyNot checkArithmeticDeref "(( ${!var} ))" +prop_checkArithmeticDeref16= verifyNot checkArithmeticDeref "(( ${x+1} + ${x=42} ))" checkArithmeticDeref params t@(TA_Expansion _ [b@(T_DollarBraced id _ _)]) = unless (isException $ bracedString b) getWarning where isException [] = True - isException s = any (`elem` "/.:#%?*@$-!") s || isDigit (head s) + isException s = any (`elem` "/.:#%?*@$-!+=^,") s || isDigit (head s) getWarning = fromMaybe noWarning . msum . map warningFor $ parents params t warningFor t = case t of From 95b1185882845afe0bafed2ebc9b086af86c05be Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 22 May 2019 17:06:48 -0700 Subject: [PATCH 136/763] Inform about ineffectual ! on commands (fixes #1531) --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 81bd2fa..262849f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - Source paths: Use `-P dir1:dir2` or a `source-path=dir1` directive to specify search paths for sourced files. - json1 format like --format=json but treats tabs as single characters +- SC2251: Inform about ineffectual ! in front of commands - SC2250: Warn about variable references without braces (optional) - SC2249: Warn about `case` with missing default case (optional) - SC2248: Warn about unquoted variables without special chars (optional) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 0db41e8..36b941e 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -190,6 +190,7 @@ nodeChecks = [ ,checkInvertedStringTest ,checkRedirectionToCommand ,checkDollarQuoteParen + ,checkUselessBang ] optionalChecks = map fst optionalTreeChecks @@ -3357,5 +3358,41 @@ checkDefaultCase _ t = pg <- wordToExactPseudoGlob pat return $ pseudoGlobIsSuperSetof pg [PGMany] +prop_checkUselessBang1 = verify checkUselessBang "! true; rest" +prop_checkUselessBang2 = verify checkUselessBang "while true; do ! true; done" +prop_checkUselessBang3 = verifyNot checkUselessBang "if ! true; then true; fi" +prop_checkUselessBang4 = verifyNot checkUselessBang "( ! true )" +prop_checkUselessBang5 = verifyNot checkUselessBang "{ ! true; }" +prop_checkUselessBang6 = verifyNot checkUselessBang "x() { ! [ x ]; }" +checkUselessBang params t = mapM_ check (getNonReturningCommands t) + where + check t = + case t of + T_Banged id _ -> + info id 2251 "This ! is not on a condition and skips errexit. Use { ! ...; } to errexit, or verify usage." + _ -> return () + + -- Get all the subcommands that aren't likely to be the return value + getNonReturningCommands :: Token -> [Token] + getNonReturningCommands t = + case t of + T_Script _ _ list -> dropLast list + T_BraceGroup _ list -> dropLast list + T_Subshell _ list -> dropLast list + T_WhileExpression _ conds cmds -> dropLast conds ++ cmds + T_UntilExpression _ conds cmds -> dropLast conds ++ cmds + T_ForIn _ _ _ list -> list + T_ForArithmetic _ _ _ _ list -> list + T_Annotation _ _ t -> getNonReturningCommands t + T_IfExpression _ conds elses -> + concatMap (dropLast . fst) conds ++ concatMap snd conds ++ elses + _ -> [] + + dropLast t = + case t of + [_] -> [] + x:rest -> x : dropLast rest + _ -> [] + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) From 36bb1e785891be8156aaf3088d05988da7816377 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 22 May 2019 17:35:41 -0700 Subject: [PATCH 137/763] Mention that "-" is supported as a filename. (Fixes #1586) --- shellcheck.1.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/shellcheck.1.md b/shellcheck.1.md index 6e24d8b..3b3498a 100644 --- a/shellcheck.1.md +++ b/shellcheck.1.md @@ -29,7 +29,6 @@ will warn that decimals are not supported. + For scripts starting with `#!/bin/ksh` (or using `-s ksh`), ShellCheck will not warn at all, as `ksh` supports decimals in arithmetic contexts. - # OPTIONS **-a**,\ **--check-sourced** @@ -113,6 +112,10 @@ not warn at all, as `ksh` supports decimals in arithmetic contexts. line (plus `/dev/null`). This option allows following any file the script may `source`. +**FILES...** + +: One or more script files to check, or "-" for standard input. + # FORMATS From 07ffcb626ebfdabe7ae283cf073a59e04994dd8b Mon Sep 17 00:00:00 2001 From: Hugo Peixoto Date: Mon, 27 May 2019 10:58:16 +0100 Subject: [PATCH 138/763] SC2016: Don't trigger when using empty backticks When using '``' or '```', it should not suggest using double quotes. --- src/ShellCheck/Analytics.hs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 36b941e..4babe73 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -917,6 +917,8 @@ prop_checkSingleQuotedVariables14= verifyNot checkSingleQuotedVariables "[ -v 'b prop_checkSingleQuotedVariables15= verifyNot checkSingleQuotedVariables "git filter-branch 'test $GIT_COMMIT'" prop_checkSingleQuotedVariables16= verify checkSingleQuotedVariables "git '$a'" prop_checkSingleQuotedVariables17= verifyNot checkSingleQuotedVariables "rename 's/(.)a/$1/g' *" +prop_checkSingleQuotedVariables18= verifyNot checkSingleQuotedVariables "echo '``'" +prop_checkSingleQuotedVariables19= verifyNot checkSingleQuotedVariables "echo '```'" checkSingleQuotedVariables params t@(T_SingleQuoted id s) = when (s `matches` re) $ @@ -962,7 +964,7 @@ checkSingleQuotedVariables params t@(T_SingleQuoted id s) = TC_Unary _ _ "-v" _ -> True _ -> False - re = mkRegex "\\$[{(0-9a-zA-Z_]|`.*`" + re = mkRegex "\\$[{(0-9a-zA-Z_]|`[^`]+`" sedContra = mkRegex "\\$[{dpsaic]($|[^a-zA-Z])" getFindCommand (T_SimpleCommand _ _ words) = From 3e7c2bfec04ed6a0479a5f0e35d129c8089f3e7e Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 2 Jun 2019 09:24:53 -0700 Subject: [PATCH 139/763] Warn about [ $a != x ] || [ $a != y ] --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 29 +++++++++++++++++++++++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 262849f..93f83c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - Source paths: Use `-P dir1:dir2` or a `source-path=dir1` directive to specify search paths for sourced files. - json1 format like --format=json but treats tabs as single characters +- SC2252: Warn about `[ $a != x ] || [ $a != y ]`, similar to SC2055 - SC2251: Inform about ineffectual ! in front of commands - SC2250: Warn about variable references without braces (optional) - SC2249: Warn about `case` with missing default case (optional) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 36b941e..3ee454c 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1348,14 +1348,39 @@ prop_checkOrNeq2 = verify checkOrNeq "(( a!=lol || a!=foo ))" prop_checkOrNeq3 = verify checkOrNeq "[ \"$a\" != lol || \"$a\" != foo ]" prop_checkOrNeq4 = verifyNot checkOrNeq "[ a != $cow || b != $foo ]" prop_checkOrNeq5 = verifyNot checkOrNeq "[[ $a != /home || $a != */public_html/* ]]" +prop_checkOrNeq6 = verify checkOrNeq "[ $a != a ] || [ $a != b ]" +prop_checkOrNeq7 = verify checkOrNeq "[ $a != a ] || [ $a != b ] || true" +prop_checkOrNeq8 = verifyNot checkOrNeq "[[ $a != x || $a != x ]]" -- This only catches the most idiomatic cases. Fixme? + +-- For test-level "or": [ x != y -o x != z ] checkOrNeq _ (TC_Or id typ op (TC_Binary _ _ op1 lhs1 rhs1 ) (TC_Binary _ _ op2 lhs2 rhs2)) - | lhs1 == lhs2 && (op1 == op2 && (op1 == "-ne" || op1 == "!=")) && not (any isGlob [rhs1,rhs2]) = + | (op1 == op2 && (op1 == "-ne" || op1 == "!=")) && lhs1 == lhs2 && rhs1 /= rhs2 && not (any isGlob [rhs1,rhs2]) = warn id 2055 $ "You probably wanted " ++ (if typ == SingleBracket then "-a" else "&&") ++ " here." +-- For arithmetic context "or" checkOrNeq _ (TA_Binary id "||" (TA_Binary _ "!=" word1 _) (TA_Binary _ "!=" word2 _)) | word1 == word2 = - warn id 2056 "You probably wanted && here." + warn id 2056 "You probably wanted && here, otherwise it's always true." + +-- For command level "or": [ x != y ] || [ x != z ] +checkOrNeq _ (T_OrIf id lhs rhs) = potentially $ do + (lhs1, op1, rhs1) <- getExpr lhs + (lhs2, op2, rhs2) <- getExpr rhs + guard $ op1 == op2 && op1 `elem` ["-ne", "!="] + guard $ lhs1 == lhs2 && rhs1 /= rhs2 + guard . not $ any isGlob [rhs1, rhs2] + return $ warn id 2252 "You probably wanted && here, otherwise it's always true." + where + getExpr x = + case x of + T_OrIf _ lhs _ -> getExpr lhs -- Fetches x and y in `T_OrIf x (T_OrIf y z)` + T_Pipeline _ _ [x] -> getExpr x + T_Redirecting _ _ c -> getExpr c + T_Condition _ _ c -> getExpr c + TC_Binary _ _ op lhs rhs -> return (lhs, op, rhs) + _ -> fail "" + checkOrNeq _ _ = return () From f4be53eb199fab001bd7460026e971103f7b8f94 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 2 Jun 2019 10:28:20 -0700 Subject: [PATCH 140/763] Warn about [ -v var ] for POSIX sh --- src/ShellCheck/Checks/ShellSupport.hs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 5ca44a1..4a86891 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -174,6 +174,7 @@ prop_checkBashisms90 = verifyNot checkBashisms "#!/bin/sh\nset -o \"$opt\"" prop_checkBashisms91 = verify checkBashisms "#!/bin/sh\nwait -n" prop_checkBashisms92 = verify checkBashisms "#!/bin/sh\necho $((16#FF))" prop_checkBashisms93 = verify checkBashisms "#!/bin/sh\necho $(( 10#$(date +%m) ))" +prop_checkBashisms94 = verify checkBashisms "#!/bin/sh\n[ -v var ]" checkBashisms = ForShell [Sh, Dash] $ \t -> do params <- ask kludge params t @@ -208,6 +209,8 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do warnMsg id "== in place of = is" bashism (TC_Binary id SingleBracket "=~" _ _) = warnMsg id "=~ regex matching is" + bashism (TC_Unary id SingleBracket "-v" _) = + warnMsg id "unary -v (in place of [ -n \"${var+x}\" ]) is" bashism (TC_Unary id _ "-a" _) = warnMsg id "unary -a in place of -e is" bashism (TA_Unary id op _) From 1297ef46d702daab795449e76f7c7f8e4b08f61d Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 2 Jun 2019 10:28:37 -0700 Subject: [PATCH 141/763] Add JSON1 as a separate format, wrap result in an object --- ShellCheck.cabal | 1 + shellcheck.hs | 5 +++-- src/ShellCheck/Formatter/JSON.hs | 17 +++++------------ 3 files changed, 9 insertions(+), 14 deletions(-) diff --git a/ShellCheck.cabal b/ShellCheck.cabal index 099052e..d54f0b8 100644 --- a/ShellCheck.cabal +++ b/ShellCheck.cabal @@ -80,6 +80,7 @@ library ShellCheck.Formatter.CheckStyle ShellCheck.Formatter.GCC ShellCheck.Formatter.JSON + ShellCheck.Formatter.JSON1 ShellCheck.Formatter.TTY ShellCheck.Formatter.Quiet ShellCheck.Interface diff --git a/shellcheck.hs b/shellcheck.hs index 02ed88a..351e1c2 100644 --- a/shellcheck.hs +++ b/shellcheck.hs @@ -27,6 +27,7 @@ import qualified ShellCheck.Formatter.CheckStyle import ShellCheck.Formatter.Format import qualified ShellCheck.Formatter.GCC import qualified ShellCheck.Formatter.JSON +import qualified ShellCheck.Formatter.JSON1 import qualified ShellCheck.Formatter.TTY import qualified ShellCheck.Formatter.Quiet @@ -141,8 +142,8 @@ formats :: FormatterOptions -> Map.Map String (IO Formatter) formats options = Map.fromList [ ("checkstyle", ShellCheck.Formatter.CheckStyle.format), ("gcc", ShellCheck.Formatter.GCC.format), - ("json", ShellCheck.Formatter.JSON.format False), -- JSON with 8-char tabs - ("json1", ShellCheck.Formatter.JSON.format True), -- JSON with 1-char tabs + ("json", ShellCheck.Formatter.JSON.format), + ("json1", ShellCheck.Formatter.JSON1.format), ("tty", ShellCheck.Formatter.TTY.format options), ("quiet", ShellCheck.Formatter.Quiet.format options) ] diff --git a/src/ShellCheck/Formatter/JSON.hs b/src/ShellCheck/Formatter/JSON.hs index c3f3219..7c26421 100644 --- a/src/ShellCheck/Formatter/JSON.hs +++ b/src/ShellCheck/Formatter/JSON.hs @@ -30,12 +30,12 @@ import GHC.Exts import System.IO import qualified Data.ByteString.Lazy.Char8 as BL -format :: Bool -> IO Formatter -format removeTabs = do +format :: IO Formatter +format = do ref <- newIORef [] return Formatter { header = return (), - onResult = collectResult removeTabs ref, + onResult = collectResult ref, onFailure = outputError, footer = finish ref } @@ -98,19 +98,12 @@ instance ToJSON Fix where outputError file msg = hPutStrLn stderr $ file ++ ": " ++ msg -collectResult removeTabs ref cr sys = mapM_ f groups +collectResult ref cr sys = mapM_ f groups where comments = crComments cr groups = groupWith sourceFile comments f :: [PositionedComment] -> IO () - f group = do - let filename = sourceFile (head group) - result <- siReadFile sys filename - let contents = either (const "") id result - let comments' = if removeTabs - then makeNonVirtual comments contents - else comments - modifyIORef ref (\x -> comments' ++ x) + f group = modifyIORef ref (\x -> comments ++ x) finish ref = do list <- readIORef ref From 9f0ef5983afddabc2bf5c5f62b11471e651224fc Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 2 Jun 2019 10:25:22 -0700 Subject: [PATCH 142/763] Optionally check for unassigned uppercase variables --- CHANGELOG.md | 1 + shellcheck.1.md | 3 +++ src/ShellCheck/Analytics.hs | 17 +++++++++++++++-- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 93f83c8..0e812ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - Source paths: Use `-P dir1:dir2` or a `source-path=dir1` directive to specify search paths for sourced files. - json1 format like --format=json but treats tabs as single characters +- SC2154: Also warn about unassigned uppercase variables (optional) - SC2252: Warn about `[ $a != x ] || [ $a != y ]`, similar to SC2055 - SC2251: Inform about ineffectual ! in front of commands - SC2250: Warn about variable references without braces (optional) diff --git a/shellcheck.1.md b/shellcheck.1.md index 3b3498a..18abb58 100644 --- a/shellcheck.1.md +++ b/shellcheck.1.md @@ -251,6 +251,9 @@ Here is an example `.shellcheckrc`: # Turn on warnings for unquoted variables with safe values enable=quote-safe-variables + # Turn on warnings for unassigned uppercase variables + enable=check-unassigned-uppercase + # Allow using `which` since it gives full paths and is common enough disable=SC2230 diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 3ee454c..cd1281c 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -231,6 +231,13 @@ optionalTreeChecks = [ cdPositive = "var=hello; echo $var", cdNegative = "var=hello; echo ${var}" }, nodeChecksToTreeCheck [checkVariableBraces]) + + ,(newCheckDescription { + cdName = "check-unassigned-uppercase", + cdDescription = "Warn when uppercase variables are unassigned", + cdPositive = "echo $VAR", + cdNegative = "VAR=hello; echo $VAR" + }, checkUnassignedReferences' True) ] optionalCheckMap :: Map.Map String (Parameters -> Token -> [TokenComment]) @@ -2131,7 +2138,10 @@ prop_checkUnassignedReferences34= verifyNotTree checkUnassignedReferences "decla prop_checkUnassignedReferences35= verifyNotTree checkUnassignedReferences "echo ${arr[foo-bar]:?fail}" prop_checkUnassignedReferences36= verifyNotTree checkUnassignedReferences "read -a foo -r <<<\"foo bar\"; echo \"$foo\"" prop_checkUnassignedReferences37= verifyNotTree checkUnassignedReferences "var=howdy; printf -v 'array[0]' %s \"$var\"; printf %s \"${array[0]}\";" -checkUnassignedReferences params t = warnings +prop_checkUnassignedReferences38= verifyTree (checkUnassignedReferences' True) "echo $VAR" + +checkUnassignedReferences = checkUnassignedReferences' False +checkUnassignedReferences' includeGlobals params t = warnings where (readMap, writeMap) = execState (mapM tally $ variableFlow params) (Map.empty, Map.empty) defaultAssigned = Map.fromList $ map (\a -> (a, ())) $ filter (not . null) internalVariables @@ -2176,8 +2186,11 @@ checkUnassignedReferences params t = warnings return $ " (did you mean '" ++ match ++ "'?)" warningFor var place = do + guard $ isVariableName var guard . not $ isInArray var place || isGuarded place - (if isLocal var then warningForLocals else warningForGlobals) var place + (if includeGlobals || isLocal var + then warningForLocals + else warningForGlobals) var place warnings = execWriter . sequence $ mapMaybe (uncurry warningFor) unassigned From 61d2112e71e7f02f44a44fd82de54b9b4b455990 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 2 Jun 2019 13:00:38 -0700 Subject: [PATCH 143/763] Add missing JSON1.hs --- src/ShellCheck/Formatter/JSON1.hs | 127 ++++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 src/ShellCheck/Formatter/JSON1.hs diff --git a/src/ShellCheck/Formatter/JSON1.hs b/src/ShellCheck/Formatter/JSON1.hs new file mode 100644 index 0000000..7335d8c --- /dev/null +++ b/src/ShellCheck/Formatter/JSON1.hs @@ -0,0 +1,127 @@ +{-# LANGUAGE OverloadedStrings #-} +{- + Copyright 2012-2019 Vidar Holen + + This file is part of ShellCheck. + https://www.shellcheck.net + + ShellCheck is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + ShellCheck is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +-} +module ShellCheck.Formatter.JSON1 (format) where + +import ShellCheck.Interface +import ShellCheck.Formatter.Format + +import Data.Aeson +import Data.IORef +import Data.Monoid +import GHC.Exts +import System.IO +import qualified Data.ByteString.Lazy.Char8 as BL + +format :: IO Formatter +format = do + ref <- newIORef [] + return Formatter { + header = return (), + onResult = collectResult ref, + onFailure = outputError, + footer = finish ref + } + +data Json1Output = Json1Output { + comments :: [PositionedComment] + } + +instance ToJSON Json1Output where + toJSON result = object [ + "comments" .= comments result + ] + toEncoding result = pairs ( + "comments" .= comments result + ) + +instance ToJSON Replacement where + toJSON replacement = + let start = repStartPos replacement + end = repEndPos replacement + str = repString replacement in + object [ + "precedence" .= repPrecedence replacement, + "insertionPoint" .= + case repInsertionPoint replacement of + InsertBefore -> "beforeStart" :: String + InsertAfter -> "afterEnd", + "line" .= posLine start, + "column" .= posColumn start, + "endLine" .= posLine end, + "endColumn" .= posColumn end, + "replacement" .= str + ] + +instance ToJSON PositionedComment where + toJSON comment = + let start = pcStartPos comment + end = pcEndPos comment + c = pcComment comment in + object [ + "file" .= posFile start, + "line" .= posLine start, + "endLine" .= posLine end, + "column" .= posColumn start, + "endColumn" .= posColumn end, + "level" .= severityText comment, + "code" .= cCode c, + "message" .= cMessage c, + "fix" .= pcFix comment + ] + + toEncoding comment = + let start = pcStartPos comment + end = pcEndPos comment + c = pcComment comment in + pairs ( + "file" .= posFile start + <> "line" .= posLine start + <> "endLine" .= posLine end + <> "column" .= posColumn start + <> "endColumn" .= posColumn end + <> "level" .= severityText comment + <> "code" .= cCode c + <> "message" .= cMessage c + <> "fix" .= pcFix comment + ) + +instance ToJSON Fix where + toJSON fix = object [ + "replacements" .= fixReplacements fix + ] + +outputError file msg = hPutStrLn stderr $ file ++ ": " ++ msg + +collectResult ref cr sys = mapM_ f groups + where + comments = crComments cr + groups = groupWith sourceFile comments + f :: [PositionedComment] -> IO () + f group = do + let filename = sourceFile (head group) + result <- siReadFile sys filename + let contents = either (const "") id result + let comments' = makeNonVirtual comments contents + modifyIORef ref (\x -> comments' ++ x) + +finish ref = do + list <- readIORef ref + BL.putStrLn $ encode $ Json1Output { comments = list } From c6dcb4127a61c4ca831907b889bb2242777b1118 Mon Sep 17 00:00:00 2001 From: Oleg Andreyev Date: Sun, 9 Jun 2019 17:00:51 +0300 Subject: [PATCH 144/763] #1607 fixing brew command --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6b789cd..3baca68 100644 --- a/README.md +++ b/README.md @@ -258,7 +258,7 @@ ShellCheck is built and packaged using Cabal. Install the package `cabal-install On MacOS (OS X), you can do a fast install of Cabal using brew, which takes a couple of minutes instead of more than 30 minutes if you try to compile it from source. brew install cask - brew cask install haskell-platform + brew cask install haskell-for-mac cabal install cabal-install On MacPorts, the package is instead called `hs-cabal-install`, while native Windows users should install the latest version of the Haskell platform from From 200aabb63c61d530c8ae8283832e92f745568ec4 Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Tue, 18 Jun 2019 23:18:26 +0200 Subject: [PATCH 145/763] Add .dockerignore This explicitly defines included/copied files, to reduce the context being sent to the Docker daemon initially. --- .dockerignore | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..39d8893 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +* +!LICENSE +!Setup.hs +!ShellCheck.cabal +!shellcheck.hs +!src From 7e77bfae491ba577aa2b859b39ea9a64cada686d Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 11 Jun 2019 18:39:09 -0700 Subject: [PATCH 146/763] Improve message for SC2055 --- src/ShellCheck/Analytics.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 1b28c6f..c2bcc78 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1365,7 +1365,7 @@ prop_checkOrNeq8 = verifyNot checkOrNeq "[[ $a != x || $a != x ]]" -- For test-level "or": [ x != y -o x != z ] checkOrNeq _ (TC_Or id typ op (TC_Binary _ _ op1 lhs1 rhs1 ) (TC_Binary _ _ op2 lhs2 rhs2)) | (op1 == op2 && (op1 == "-ne" || op1 == "!=")) && lhs1 == lhs2 && rhs1 /= rhs2 && not (any isGlob [rhs1,rhs2]) = - warn id 2055 $ "You probably wanted " ++ (if typ == SingleBracket then "-a" else "&&") ++ " here." + warn id 2055 $ "You probably wanted " ++ (if typ == SingleBracket then "-a" else "&&") ++ " here, otherwise it's always true." -- For arithmetic context "or" checkOrNeq _ (TA_Binary id "||" (TA_Binary _ "!=" word1 _) (TA_Binary _ "!=" word2 _)) From 5242e384a11ce6855a4be8b23944634ee60fccfd Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 23 Jun 2019 13:47:35 -0700 Subject: [PATCH 147/763] Fix error spans for shebang warnings (fixes #1620) --- src/ShellCheck/AST.hs | 2 +- src/ShellCheck/Analytics.hs | 4 ++-- src/ShellCheck/AnalyzerLib.hs | 4 ++-- src/ShellCheck/Parser.hs | 24 ++++++++++++++---------- 4 files changed, 19 insertions(+), 15 deletions(-) diff --git a/src/ShellCheck/AST.hs b/src/ShellCheck/AST.hs index eb236ca..d8faec6 100644 --- a/src/ShellCheck/AST.hs +++ b/src/ShellCheck/AST.hs @@ -121,7 +121,7 @@ data Token = | T_Rbrace Id | T_Redirecting Id [Token] Token | T_Rparen Id - | T_Script Id String [Token] + | T_Script Id Token [Token] -- Shebang T_Literal, followed by script. | T_Select Id | T_SelectIn Id String [Token] [Token] | T_Semi Id diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index c2bcc78..e1523fd 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -534,7 +534,7 @@ indexOfSublists sub = f 0 prop_checkShebangParameters1 = verifyTree checkShebangParameters "#!/usr/bin/env bash -x\necho cow" prop_checkShebangParameters2 = verifyNotTree checkShebangParameters "#! /bin/sh -l " checkShebangParameters p (T_Annotation _ _ t) = checkShebangParameters p t -checkShebangParameters _ (T_Script id sb _) = +checkShebangParameters _ (T_Script _ (T_Literal id sb) _) = [makeComment ErrorC id 2096 "On most OS, shebangs can only specify a single parameter." | length (words sb) > 2] prop_checkShebang1 = verifyNotTree checkShebang "#!/usr/bin/env bash -x\necho cow" @@ -554,7 +554,7 @@ checkShebang params (T_Annotation _ list t) = where isOverride (ShellOverride _) = True isOverride _ = False -checkShebang params (T_Script id sb _) = execWriter $ do +checkShebang params (T_Script _ (T_Literal id sb) _) = execWriter $ do unless (shellTypeSpecified params) $ do when (sb == "") $ err id 2148 "Tips depend on target shell and yours is unknown. Add a shebang." diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index d99ea98..388f871 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -206,7 +206,7 @@ containsSetE root = isNothing $ doAnalysis (guard . not . isSetE) root where isSetE t = case t of - T_Script _ str _ -> str `matches` re + T_Script _ (T_Literal _ str) _ -> str `matches` re T_SimpleCommand {} -> t `isUnqualifiedCommand` "set" && ("errexit" `elem` oversimplify t || @@ -252,7 +252,7 @@ determineShell fallbackShell t = fromMaybe Bash $ do getCandidates (T_Annotation _ annotations s) = map forAnnotation annotations ++ [Just $ fromShebang s] - fromShebang (T_Script _ s t) = executableFromShebang s + fromShebang (T_Script _ (T_Literal _ s) _) = executableFromShebang s -- Given a string like "/bin/bash" or "/usr/bin/env dash", -- return the shell basename like "bash" or "dash" diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index c09d64c..cd4bc8f 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -2790,10 +2790,11 @@ readAssignmentWordExt lenient = try $ do string "=" >> return Assign ] - readEmptyLiteral = do - start <- startSpan - id <- endSpan start - return $ T_Literal id "" + +readEmptyLiteral = do + start <- startSpan + id <- endSpan start + return $ T_Literal id "" readArrayIndex = do start <- startSpan @@ -2941,12 +2942,14 @@ prop_readShebang5 = isWarning readShebang "\n#!/bin/sh" prop_readShebang6 = isWarning readShebang " # Copyright \n!#/bin/bash" prop_readShebang7 = isNotOk readShebang "# Copyright \nfoo\n#!/bin/bash" readShebang = do + start <- startSpan anyShebang <|> try readMissingBang <|> withHeader many linewhitespace str <- many $ noneOf "\r\n" + id <- endSpan start optional carriageReturn optional linefeed - return str + return $ T_Literal id str where anyShebang = choice $ map try [ readCorrect, @@ -3077,7 +3080,8 @@ readScriptFile sourced = do readUtf8Bom parseProblem ErrorC 1082 "This file has a UTF-8 BOM. Remove it with: LC_CTYPE=C sed '1s/^...//' < yourscript ." - sb <- option "" readShebang + shebang <- readShebang <|> readEmptyLiteral + let (T_Literal _ shebangString) = shebang allspacing annotationStart <- startSpan fileAnnotations <- readAnnotations @@ -3094,19 +3098,19 @@ readScriptFile sourced = do let ignoreShebang = shellAnnotationSpecified || shellFlagSpecified unless ignoreShebang $ - verifyShebang pos (getShell sb) - if ignoreShebang || isValidShell (getShell sb) /= Just False + verifyShebang pos (getShell shebangString) + if ignoreShebang || isValidShell (getShell shebangString) /= Just False then do commands <- withAnnotations annotations readCompoundListOrEmpty id <- endSpan start verifyEof let script = T_Annotation annotationId annotations $ - T_Script id sb commands + T_Script id shebang commands reparseIndices script else do many anyChar id <- endSpan start - return $ T_Script id sb [] + return $ T_Script id shebang [] where basename s = reverse . takeWhile (/= '/') . reverse $ s From e099625e7d09f26146d0197d6ff466dc8cd89b39 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 23 Jun 2019 14:26:18 -0700 Subject: [PATCH 148/763] Remove unused ioref --- src/ShellCheck/Formatter/Quiet.hs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/ShellCheck/Formatter/Quiet.hs b/src/ShellCheck/Formatter/Quiet.hs index 9ad8b97..b7e0ee9 100644 --- a/src/ShellCheck/Formatter/Quiet.hs +++ b/src/ShellCheck/Formatter/Quiet.hs @@ -27,8 +27,7 @@ import Data.IORef import System.Exit format :: FormatterOptions -> IO Formatter -format options = do - topErrorRef <- newIORef [] +format options = return Formatter { header = return (), footer = return (), From b8b4a11348bb77514c77679c2a58a721fc89f31e Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 23 Jun 2019 19:18:45 -0700 Subject: [PATCH 149/763] Update JSON1 docs in man page --- shellcheck.1.md | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/shellcheck.1.md b/shellcheck.1.md index 18abb58..c9963bd 100644 --- a/shellcheck.1.md +++ b/shellcheck.1.md @@ -158,22 +158,24 @@ not warn at all, as `ksh` supports decimals in arithmetic contexts. applications. ShellCheck's json is compact and contains only the bare minimum. Tabs are counted as 1 character. - [ - { - "file": "filename", - "line": lineNumber, - "column": columnNumber, - "level": "severitylevel", - "code": errorCode, - "message": "warning message" - }, - ... - ] + { + comments: [ + { + "file": "filename", + "line": lineNumber, + "column": columnNumber, + "level": "severitylevel", + "code": errorCode, + "message": "warning message" + }, + ... + ] + } **json** -: This is a legacy version of the **json1** format, with a tab stop - of 8 instead of 1. +: This is a legacy version of the **json1** format. It's a raw array of + comments, and all offsets have a tab stop of 8. **quiet** From b1aeee564c6852147081dda08b1030e864b1711f Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 23 Jun 2019 19:05:12 -0700 Subject: [PATCH 150/763] Add a Diff output format --- CHANGELOG.md | 1 + ShellCheck.cabal | 4 + shellcheck.1.md | 17 +++ shellcheck.hs | 2 + src/ShellCheck/Formatter/Diff.hs | 222 +++++++++++++++++++++++++++++ src/ShellCheck/Formatter/Format.hs | 17 ++- src/ShellCheck/Formatter/TTY.hs | 8 +- test/shellcheck.hs | 2 + 8 files changed, 265 insertions(+), 8 deletions(-) create mode 100644 src/ShellCheck/Formatter/Diff.hs diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e812ac..e074894 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## Since previous release ### Added - Preliminary support for fix suggestions +- New `-f diff` unified diff format for auto-fixes - Files containing Bats tests can now be checked - Directory wide directives can now be placed in a `.shellcheckrc` - Optional checks: Use `--list-optional` to show a list of tests, diff --git a/ShellCheck.cabal b/ShellCheck.cabal index d54f0b8..781a082 100644 --- a/ShellCheck.cabal +++ b/ShellCheck.cabal @@ -57,6 +57,7 @@ library bytestring, containers >= 0.5, deepseq >= 1.4.0.0, + Diff >= 0.2.0, directory >= 1.2.3.0, mtl >= 2.2.1, filepath, @@ -78,6 +79,7 @@ library ShellCheck.Fixer ShellCheck.Formatter.Format ShellCheck.Formatter.CheckStyle + ShellCheck.Formatter.Diff ShellCheck.Formatter.GCC ShellCheck.Formatter.JSON ShellCheck.Formatter.JSON1 @@ -100,6 +102,7 @@ executable shellcheck bytestring, containers, deepseq >= 1.4.0.0, + Diff >= 0.2.0, directory >= 1.2.3.0, mtl >= 2.2.1, filepath, @@ -118,6 +121,7 @@ test-suite test-shellcheck bytestring, containers, deepseq >= 1.4.0.0, + Diff >= 0.2.0, directory >= 1.2.3.0, mtl >= 2.2.1, filepath, diff --git a/shellcheck.1.md b/shellcheck.1.md index c9963bd..77fa79a 100644 --- a/shellcheck.1.md +++ b/shellcheck.1.md @@ -152,6 +152,23 @@ not warn at all, as `ksh` supports decimals in arithmetic contexts. ... +**diff** + +: Auto-fixes in unified diff format. Can be piped to `git apply` or `patch -p1` + to automatically apply fixes. + + --- a/test.sh + +++ b/test.sh + @@ -2,6 +2,6 @@ + ## Example of a broken script. + for f in $(ls *.m3u) + do + - grep -qi hq.*mp3 $f \ + + grep -qi hq.*mp3 "$f" \ + && echo -e 'Playlist $f contains a HQ file in mp3 format' + done + + **json1** : Json is a popular serialization format that is more suitable for web diff --git a/shellcheck.hs b/shellcheck.hs index 351e1c2..4ba8b70 100644 --- a/shellcheck.hs +++ b/shellcheck.hs @@ -25,6 +25,7 @@ import ShellCheck.Regex import qualified ShellCheck.Formatter.CheckStyle import ShellCheck.Formatter.Format +import qualified ShellCheck.Formatter.Diff import qualified ShellCheck.Formatter.GCC import qualified ShellCheck.Formatter.JSON import qualified ShellCheck.Formatter.JSON1 @@ -141,6 +142,7 @@ parseArguments argv = formats :: FormatterOptions -> Map.Map String (IO Formatter) formats options = Map.fromList [ ("checkstyle", ShellCheck.Formatter.CheckStyle.format), + ("diff", ShellCheck.Formatter.Diff.format options), ("gcc", ShellCheck.Formatter.GCC.format), ("json", ShellCheck.Formatter.JSON.format), ("json1", ShellCheck.Formatter.JSON1.format), diff --git a/src/ShellCheck/Formatter/Diff.hs b/src/ShellCheck/Formatter/Diff.hs new file mode 100644 index 0000000..f89a756 --- /dev/null +++ b/src/ShellCheck/Formatter/Diff.hs @@ -0,0 +1,222 @@ +{- + Copyright 2019 Vidar 'koala_man' Holen + + This file is part of ShellCheck. + https://www.shellcheck.net + + ShellCheck is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + ShellCheck is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +-} +{-# LANGUAGE TemplateHaskell #-} +module ShellCheck.Formatter.Diff (format, ShellCheck.Formatter.Diff.runTests) where + +import ShellCheck.Interface +import ShellCheck.Fixer +import ShellCheck.Formatter.Format + +import Control.Monad +import Data.Algorithm.Diff +import Data.Array +import Data.IORef +import Data.List +import Data.Maybe +import qualified Data.Map as M +import GHC.Exts (sortWith) +import System.IO +import System.FilePath + +import Test.QuickCheck + +import Debug.Trace +ltt x = trace (show x) x + +format :: FormatterOptions -> IO Formatter +format options = do + didOutput <- newIORef False + shouldColor <- shouldOutputColor (foColorOption options) + let color = if shouldColor then colorize else nocolor + return Formatter { + header = return (), + footer = checkFooter didOutput color, + onFailure = reportFailure color, + onResult = reportResult didOutput color + } + + +contextSize = 3 +red = 31 +green = 32 +yellow = 33 +cyan = 36 +bold = 1 + +nocolor n = id +colorize n s = (ansi n) ++ s ++ (ansi 0) +ansi n = "\x1B[" ++ show n ++ "m" + +printErr :: ColorFunc -> String -> IO () +printErr color = hPutStrLn stderr . color bold . color red +reportFailure color file msg = printErr color $ file ++ ": " ++ msg + +checkFooter didOutput color = do + output <- readIORef didOutput + unless output $ + printErr color "Issues were detected, but none were auto-fixable. Use another format to see them." + +type ColorFunc = (Int -> String -> String) +data DiffDoc a = DiffDoc String [DiffRegion a] +data DiffRegion a = DiffRegion (Int, Int) (Int, Int) [Diff a] + +reportResult :: (IORef Bool) -> ColorFunc -> CheckResult -> SystemInterface IO -> IO () +reportResult didOutput color result sys = do + let comments = crComments result + let suggestedFixes = mapMaybe pcFix comments + let fixmap = buildFixMap suggestedFixes + mapM_ output $ M.toList fixmap + where + output (name, fix) = do + file <- (siReadFile sys) name + case file of + Right contents -> do + putStrLn $ formatDoc color $ makeDiff name contents fix + writeIORef didOutput True + Left msg -> reportFailure color name msg + +makeDiff :: String -> String -> Fix -> DiffDoc String +makeDiff name contents fix = + DiffDoc name $ findRegions . groupDiff $ computeDiff contents fix + +computeDiff :: String -> Fix -> [Diff String] +computeDiff contents fix = + let old = lines contents + array = listArray (1, fromIntegral $ (length old)) old + new = applyFix fix array + in getDiff old new + +-- Group changes into hunks +groupDiff :: [Diff a] -> [(Bool, [Diff a])] +groupDiff = filter (\(_, l) -> not (null l)) . hunt [] + where + -- Churn through 'Both's until we find a difference + hunt current [] = [(False, reverse current)] + hunt current (x@Both {}:rest) = hunt (x:current) rest + hunt current list = + let (context, previous) = splitAt contextSize current + in (False, reverse previous) : gather context 0 list + + -- Pick out differences until we find a run of Both's + gather current n [] = + let (extras, patch) = splitAt (max 0 $ n - contextSize) current + in [(True, reverse patch), (False, reverse extras)] + + gather current n list@(Both {}:_) | n == contextSize*2 = + let (context, previous) = splitAt contextSize current + in (True, reverse previous) : hunt context list + + gather current n (x@Both {}:rest) = gather (x:current) (n+1) rest + gather current n (x:rest) = gather (x:current) 0 rest + +-- Get line numbers for hunks +findRegions :: [(Bool, [Diff String])] -> [DiffRegion String] +findRegions = find' 1 1 + where + find' _ _ [] = [] + find' left right ((output, run):rest) = + let (dl, dr) = countDelta run + remainder = find' (left+dl) (right+dr) rest + in + if output + then DiffRegion (left, dl) (right, dr) run : remainder + else remainder + +-- Get left/right line counts for a hunk +countDelta :: [Diff a] -> (Int, Int) +countDelta = count' 0 0 + where + count' left right [] = (left, right) + count' left right (x:rest) = + case x of + Both {} -> count' (left+1) (right+1) rest + First {} -> count' (left+1) right rest + Second {} -> count' left (right+1) rest + +formatRegion :: ColorFunc -> DiffRegion String -> String +formatRegion color (DiffRegion left right diffs) = + let header = color cyan ("@@ -" ++ (tup left) ++ " +" ++ (tup right) ++" @@") + in + unlines $ header : map format diffs + where + tup (a,b) = (show a) ++ "," ++ (show b) + format (Both x _) = ' ':x + format (First x) = color red $ '-':x + format (Second x) = color green $ '+':x + +formatDoc color (DiffDoc name regions) = + (color bold $ "--- " ++ ("a" name)) ++ "\n" ++ + (color bold $ "+++ " ++ ("b" name)) ++ "\n" ++ + concatMap (formatRegion color) regions + +-- Create a Map from filename to Fix +buildFixMap :: [Fix] -> M.Map String Fix +buildFixMap fixes = perFile + where + splitFixes = concatMap splitFixByFile fixes + perFile = groupByMap (posFile . repStartPos . head . fixReplacements) splitFixes + +-- There are currently no multi-file fixes, but let's handle it anyways +splitFixByFile :: Fix -> [Fix] +splitFixByFile fix = map makeFix $ groupBy sameFile (fixReplacements fix) + where + sameFile rep1 rep2 = (posFile $ repStartPos rep1) == (posFile $ repStartPos rep2) + makeFix reps = newFix { fixReplacements = reps } + +groupByMap :: (Ord k, Monoid v) => (v -> k) -> [v] -> M.Map k v +groupByMap f = M.fromListWith (<>) . map (\x -> (f x, x)) + +-- For building unit tests +b n = Both n n +l = First +r = Second + +prop_identifiesProperContext = groupDiff [b 1, b 2, b 3, b 4, l 5, b 6, b 7, b 8, b 9] == + [(False, [b 1]), -- Omitted + (True, [b 2, b 3, b 4, l 5, b 6, b 7, b 8]), -- A change with three lines of context + (False, [b 9])] -- Omitted + +prop_includesContextFromStartIfNecessary = groupDiff [b 4, l 5, b 6, b 7, b 8, b 9] == + [ -- Nothing omitted + (True, [b 4, l 5, b 6, b 7, b 8]), -- A change with three lines of context + (False, [b 9])] -- Omitted + +prop_includesContextUntilEndIfNecessary = groupDiff [b 4, l 5] == + [ -- Nothing omitted + (True, [b 4, l 5]) + ] -- Nothing Omitted + +prop_splitsIntoMultipleHunks = groupDiff [l 1, b 1, b 2, b 3, b 4, b 5, b 6, b 7, r 8] == + [ -- Nothing omitted + (True, [l 1, b 1, b 2, b 3]), + (False, [b 4]), + (True, [b 5, b 6, b 7, r 8]) + ] -- Nothing Omitted + +prop_splitsIntoMultipleHunksUnlessTouching = groupDiff [l 1, b 1, b 2, b 3, b 4, b 5, b 6, r 7] == + [ + (True, [l 1, b 1, b 2, b 3, b 4, b 5, b 6, r 7]) + ] + +prop_countDeltasWorks = countDelta [b 1, l 2, r 3, r 4, b 5] == (3,4) +prop_countDeltasWorks2 = countDelta [] == (0,0) + +return [] +runTests = $quickCheckAll diff --git a/src/ShellCheck/Formatter/Format.hs b/src/ShellCheck/Formatter/Format.hs index 11dfd17..cb7dfe6 100644 --- a/src/ShellCheck/Formatter/Format.hs +++ b/src/ShellCheck/Formatter/Format.hs @@ -22,8 +22,12 @@ module ShellCheck.Formatter.Format where import ShellCheck.Data import ShellCheck.Interface import ShellCheck.Fixer + import Control.Monad import Data.Array +import Data.List +import System.IO +import System.Info -- A formatter that carries along an arbitrary piece of data data Formatter = Formatter { @@ -59,6 +63,17 @@ makeNonVirtual comments contents = fixReplacements = map (\r -> removeTabStops r arr) (fixReplacements f) } fix c = (removeTabStops c arr) { - pcFix = liftM untabbedFix (pcFix c) + pcFix = fmap untabbedFix (pcFix c) } + +shouldOutputColor :: ColorOption -> IO Bool +shouldOutputColor colorOption = do + term <- hIsTerminalDevice stdout + let windows = "mingw" `isPrefixOf` os + let isUsableTty = term && not windows + let useColor = case colorOption of + ColorAlways -> True + ColorNever -> False + ColorAuto -> isUsableTty + return useColor diff --git a/src/ShellCheck/Formatter/TTY.hs b/src/ShellCheck/Formatter/TTY.hs index 845feeb..4dabf45 100644 --- a/src/ShellCheck/Formatter/TTY.hs +++ b/src/ShellCheck/Formatter/TTY.hs @@ -188,13 +188,7 @@ code num = "SC" ++ show num getColorFunc :: ColorOption -> IO ColorFunc getColorFunc colorOption = do - term <- hIsTerminalDevice stdout - let windows = "mingw" `isPrefixOf` os - let isUsableTty = term && not windows - let useColor = case colorOption of - ColorAlways -> True - ColorNever -> False - ColorAuto -> isUsableTty + useColor <- shouldOutputColor colorOption return $ if useColor then colorComment else const id where colorComment level comment = diff --git a/test/shellcheck.hs b/test/shellcheck.hs index 8f858d6..d55b140 100644 --- a/test/shellcheck.hs +++ b/test/shellcheck.hs @@ -8,6 +8,7 @@ import qualified ShellCheck.Checker import qualified ShellCheck.Checks.Commands import qualified ShellCheck.Checks.ShellSupport import qualified ShellCheck.Fixer +import qualified ShellCheck.Formatter.Diff import qualified ShellCheck.Parser main = do @@ -19,6 +20,7 @@ main = do ,ShellCheck.Checks.Commands.runTests ,ShellCheck.Checks.ShellSupport.runTests ,ShellCheck.Fixer.runTests + ,ShellCheck.Formatter.Diff.runTests ,ShellCheck.Parser.runTests ] if and results From c5aa171a5f9eefe4f6280112373e0a2f178e7af8 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 24 Jun 2019 09:02:35 -0700 Subject: [PATCH 151/763] Use mappend over <> for compatibility --- src/ShellCheck/Formatter/Diff.hs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Formatter/Diff.hs b/src/ShellCheck/Formatter/Diff.hs index f89a756..58fd2f4 100644 --- a/src/ShellCheck/Formatter/Diff.hs +++ b/src/ShellCheck/Formatter/Diff.hs @@ -29,6 +29,7 @@ import Data.Algorithm.Diff import Data.Array import Data.IORef import Data.List +import qualified Data.Monoid as Monoid import Data.Maybe import qualified Data.Map as M import GHC.Exts (sortWith) @@ -181,7 +182,7 @@ splitFixByFile fix = map makeFix $ groupBy sameFile (fixReplacements fix) makeFix reps = newFix { fixReplacements = reps } groupByMap :: (Ord k, Monoid v) => (v -> k) -> [v] -> M.Map k v -groupByMap f = M.fromListWith (<>) . map (\x -> (f x, x)) +groupByMap f = M.fromListWith Monoid.mappend . map (\x -> (f x, x)) -- For building unit tests b n = Both n n From f6ba500d6b5c80f6ec790d7bcde93418bbf064ff Mon Sep 17 00:00:00 2001 From: Benjamin Gordon Date: Fri, 31 May 2019 10:42:53 -0600 Subject: [PATCH 152/763] Add support for basic shflags semantics The shflags command-line flags library creates variables at runtime with a few well-defined functions. This causes shellcheck to spit out lots of warnings about unassigned variables, as well as miss warnings about unused flag variables. We can address this with two parts: 1. Pretend that the shflags global variables are predefined like other shell variables so that shellcheck doesn't expect users to set them. 2. Treat DEFINE_string, DEFINE_int, etc. as new commands that create variables, similar to the existing read, local, mapfile, etc. Part 1 can theoretically be addresssed without this by following sourced files, but that doesn't help if people are otherwise not following external sources. The new behavior is on by default, similar to automatic bats test behavior. Addresses #1597 --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 2 ++ src/ShellCheck/AnalyzerLib.hs | 11 +++++++++++ src/ShellCheck/Data.hs | 8 ++++++++ 4 files changed, 22 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e074894..28fbb4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ - Source paths: Use `-P dir1:dir2` or a `source-path=dir1` directive to specify search paths for sourced files. - json1 format like --format=json but treats tabs as single characters +- Recognize FLAGS variables created by the shflags library. - SC2154: Also warn about unassigned uppercase variables (optional) - SC2252: Warn about `[ $a != x ] || [ $a != y ]`, similar to SC2055 - SC2251: Inform about ineffectual ! in front of commands diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index e1523fd..9377fa0 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2081,6 +2081,8 @@ prop_checkUnused38= verifyTree checkUnusedAssignments "(( a=42 ))" prop_checkUnused39= verifyNotTree checkUnusedAssignments "declare -x -f foo" prop_checkUnused40= verifyNotTree checkUnusedAssignments "arr=(1 2); num=2; echo \"${arr[@]:num}\"" prop_checkUnused41= verifyNotTree checkUnusedAssignments "@test 'foo' {\ntrue\n}\n" +prop_checkUnused42= verifyNotTree checkUnusedAssignments "DEFINE_string foo '' ''; echo \"${FLAGS_foo}\"" +prop_checkUnused43= verifyTree checkUnusedAssignments "DEFINE_string foo '' ''" checkUnusedAssignments params t = execWriter (mapM_ warnFor unused) where flow = variableFlow params diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 388f871..508b6ee 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -606,6 +606,11 @@ getModifiedVariableCommand base@(T_SimpleCommand _ _ (T_NormalWord _ (T_Literal "mapfile" -> maybeToList $ getMapfileArray base rest "readarray" -> maybeToList $ getMapfileArray base rest + "DEFINE_boolean" -> maybeToList $ getFlagVariable rest + "DEFINE_float" -> maybeToList $ getFlagVariable rest + "DEFINE_integer" -> maybeToList $ getFlagVariable rest + "DEFINE_string" -> maybeToList $ getFlagVariable rest + _ -> [] where flags = map snd $ getAllFlags base @@ -679,6 +684,12 @@ getModifiedVariableCommand base@(T_SimpleCommand _ _ (T_NormalWord _ (T_Literal map (getLiteralArray . snd) (filter (\(x,_) -> getLiteralString x == Just "-a") (zip (args) (tail args))) + -- get the FLAGS_ variable created by a shflags DEFINE_ call + getFlagVariable (n:v:_) = return (base, n, flagName n, DataString $ SourceFrom [v]) + where + flagName varName@(T_NormalWord _ _) = "FLAGS_" ++ (onlyLiteralString varName) + getFlagVariable _ = fail "Invalid flag definition" + getModifiedVariableCommand _ = [] getIndexReferences s = fromMaybe [] $ do diff --git a/src/ShellCheck/Data.hs b/src/ShellCheck/Data.hs index 2eedeeb..1394c04 100644 --- a/src/ShellCheck/Data.hs +++ b/src/ShellCheck/Data.hs @@ -36,6 +36,11 @@ internalVariables = [ -- Ksh , ".sh.version" + + -- shflags + , "FLAGS_ARGC", "FLAGS_ARGV", "FLAGS_ERROR", "FLAGS_FALSE", "FLAGS_HELP", + "FLAGS_PARENT", "FLAGS_RESERVED", "FLAGS_TRUE", "FLAGS_VERSION", + "flags_error", "flags_return" ] specialVariablesWithoutSpaces = [ @@ -45,6 +50,9 @@ variablesWithoutSpaces = specialVariablesWithoutSpaces ++ [ "BASHPID", "BASH_ARGC", "BASH_LINENO", "BASH_SUBSHELL", "EUID", "LINENO", "OPTIND", "PPID", "RANDOM", "SECONDS", "SHELLOPTS", "SHLVL", "UID", "COLUMNS", "HISTFILESIZE", "HISTSIZE", "LINES" + + -- shflags + , "FLAGS_ERROR", "FLAGS_FALSE", "FLAGS_TRUE" ] specialVariables = specialVariablesWithoutSpaces ++ ["@", "*"] From e95d8dd14e2351fb3acb1c5b70c0e890c4c928b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Avi=20=D7=93?= Date: Sat, 29 Jun 2019 03:40:05 -0400 Subject: [PATCH 153/763] Bump stack snapshot --- stack.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stack.yaml b/stack.yaml index d39cada..6dee632 100644 --- a/stack.yaml +++ b/stack.yaml @@ -2,7 +2,7 @@ # For more information, see: https://docs.haskellstack.org/en/stable/yaml_configuration/ # Specifies the GHC version and set of packages available (e.g., lts-3.5, nightly-2015-09-21, ghc-7.10.2) -resolver: lts-8.5 +resolver: lts-13.26 # Local packages, usually specified by relative directory name packages: From 3116ed3ae59458148eee77325ce72af7a227d19f Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 30 Jun 2019 16:36:03 -0700 Subject: [PATCH 154/763] Filter warnings by annotations in unit tests --- src/ShellCheck/Analytics.hs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index e1523fd..b3f3cb2 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -273,7 +273,11 @@ producesComments :: (Parameters -> Token -> [TokenComment]) -> String -> Maybe B producesComments f s = do let pr = pScript s prRoot pr - return . not . null $ runList (defaultSpec pr) [f] + let spec = defaultSpec pr + let params = makeParameters spec + return . not . null $ + filterByAnnotation spec params $ + runList spec [f] -- Copied from https://wiki.haskell.org/Edit_distance dist :: Eq a => [a] -> [a] -> Int From eeb7ea01c95a900039abb41091c097af77a8f129 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 30 Jun 2019 16:36:45 -0700 Subject: [PATCH 155/763] Allow SC2103 to be silenced (fixes #1591) --- src/ShellCheck/Analytics.hs | 33 ++++++++++----------------------- 1 file changed, 10 insertions(+), 23 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index b3f3cb2..f89c51f 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2343,27 +2343,18 @@ prop_checkCdAndBack4 = verify checkCdAndBack "cd $tmp; foo; cd -" prop_checkCdAndBack5 = verifyNot checkCdAndBack "cd ..; foo; cd .." prop_checkCdAndBack6 = verify checkCdAndBack "for dir in */; do cd \"$dir\"; some_cmd; cd ..; done" prop_checkCdAndBack7 = verifyNot checkCdAndBack "set -e; for dir in */; do cd \"$dir\"; some_cmd; cd ..; done" -checkCdAndBack params = doLists +prop_checkCdAndBack8 = verifyNot checkCdAndBack "cd tmp\nfoo\n# shellcheck disable=SC2103\ncd ..\n" +checkCdAndBack params t = + unless (hasSetE params) $ mapM_ doList $ getCommandSequences t where - shell = shellType params - doLists (T_ForIn _ _ _ cmds) = doList cmds - doLists (T_ForArithmetic _ _ _ _ cmds) = doList cmds - doLists (T_WhileExpression _ _ cmds) = doList cmds - doLists (T_UntilExpression _ _ cmds) = doList cmds - doLists (T_Script _ _ cmds) = doList cmds - doLists (T_IfExpression _ thens elses) = do - mapM_ (\(_, l) -> doList l) thens - doList elses - doLists _ = return () - isCdRevert t = case oversimplify t of - ["cd", p] -> p `elem` ["..", "-"] + [_, p] -> p `elem` ["..", "-"] _ -> False - getCmd (T_Annotation id _ x) = getCmd x - getCmd (T_Pipeline id _ [x]) = getCommandName x - getCmd _ = Nothing + getCandidate (T_Annotation _ _ x) = getCandidate x + getCandidate (T_Pipeline id _ [x]) | x `isCommand` "cd" = return x + getCandidate _ = Nothing findCdPair list = case list of @@ -2373,13 +2364,9 @@ checkCdAndBack params = doLists else findCdPair (b:rest) _ -> Nothing - doList list = - if hasSetE params - then return () - else let cds = filter ((== Just "cd") . getCmd) list - in potentially $ do - cd <- findCdPair cds - return $ info cd 2103 "Use a ( subshell ) to avoid having to cd back." + doList list = potentially $ do + cd <- findCdPair $ mapMaybe getCandidate list + return $ info cd 2103 "Use a ( subshell ) to avoid having to cd back." prop_checkLoopKeywordScope1 = verify checkLoopKeywordScope "continue 2" prop_checkLoopKeywordScope2 = verify checkLoopKeywordScope "for f; do ( break; ); done" From c381c5746f86eb43cfce91104c47215d1563b349 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 30 Jun 2019 17:28:15 -0700 Subject: [PATCH 156/763] Remove unnecessary lookahead in readDollarLonely --- src/ShellCheck/Parser.hs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index cd4bc8f..e00e1f5 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -1698,7 +1698,6 @@ readDollarLonely = do start <- startSpan char '$' id <- endSpan start - n <- lookAhead (anyChar <|> (eof >> return '_')) return $ T_Literal id "$" prop_readHereDoc = isOk readScript "cat << foo\nlol\ncow\nfoo" From 321afa427ef8efcf34f5b2aa0d327f7d1e6715f8 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 30 Jun 2019 17:38:17 -0700 Subject: [PATCH 157/763] Remove unused parse-time AST warnings --- src/ShellCheck/Parser.hs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index e00e1f5..059f03b 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -138,7 +138,6 @@ almostSpace = return ' ' --------- Message/position annotation on top of user state -data Note = Note Id Severity Code String deriving (Show, Eq) data ParseNote = ParseNote SourcePos SourcePos Severity Code String deriving (Show, Eq) data Context = ContextName SourcePos String @@ -166,10 +165,6 @@ initialUserState = UserState { } codeForParseNote (ParseNote _ _ _ code _) = code -noteToParseNote map (Note id severity code message) = - ParseNote pos pos severity code message - where - pos = fromJust $ Map.lookup id map getLastId = lastId <$> getState From 544047c5afa19fb1551ba4f0d4f56ef34746e391 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 30 Jun 2019 18:26:41 -0700 Subject: [PATCH 158/763] Warn about ending double quotes just to make $ literal --- CHANGELOG.md | 1 + src/ShellCheck/Parser.hs | 27 ++++++++++++++++++++++++--- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e074894..5ea2768 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ - SC2246: Warn if a shebang's interpreter ends with / - SC2245: Warn that Ksh ignores all but the first glob result in `[` - SC2243/SC2244: Suggest using explicit -n for `[ $foo ]` (optional) +- SC1135: Suggest not ending double quotes just to make $ literal ### Changed - If a directive or shebang is not specified, a `.bash/.bats/.dash/.ksh` diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 059f03b..445d43b 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -1524,10 +1524,10 @@ ensureDollar = readNormalDollar = do ensureDollar - readDollarExp <|> readDollarDoubleQuote <|> readDollarSingleQuote <|> readDollarLonely + readDollarExp <|> readDollarDoubleQuote <|> readDollarSingleQuote <|> readDollarLonely False readDoubleQuotedDollar = do ensureDollar - readDollarExp <|> readDollarLonely + readDollarExp <|> readDollarLonely True prop_readDollarExpression1 = isOk readDollarExpression "$(((1) && 3))" @@ -1689,11 +1689,32 @@ readVariableName = do rest <- many variableChars return (f:rest) -readDollarLonely = do + +prop_readDollarLonely1 = isWarning readNormalWord "\"$\"var" +prop_readDollarLonely2 = isWarning readNormalWord "\"$\"\"var\"" +prop_readDollarLonely3 = isOk readNormalWord "\"$\"$var" +prop_readDollarLonely4 = isOk readNormalWord "\"$\"*" +prop_readDollarLonely5 = isOk readNormalWord "$\"str\"" +readDollarLonely quoted = do start <- startSpan char '$' id <- endSpan start + when quoted $ do + isHack <- quoteForEscape + when isHack $ + parseProblemAtId id StyleC 1135 + "Prefer escape over ending quote to make $ literal. Instead of \"It costs $\"5, use \"It costs \\$5\"." return $ T_Literal id "$" + where + quoteForEscape = option False $ try . lookAhead $ do + char '"' + -- Check for "foo $""bar" + optional $ char '"' + c <- anyVar + -- Don't trigger on [[ x == "$"* ]] or "$"$pattern + return $ c `notElem` "*$" + anyVar = variableStart <|> digit <|> specialVariable + prop_readHereDoc = isOk readScript "cat << foo\nlol\ncow\nfoo" prop_readHereDoc2 = isNotOk readScript "cat <<- EOF\n cow\n EOF" From 9702f1ff9c21e4c6508345bd9a9fbcd37de43c0a Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 30 Jun 2019 20:19:10 -0700 Subject: [PATCH 159/763] Handle diffs for files without trailing linefeed --- src/ShellCheck/Formatter/Diff.hs | 48 ++++++++++++++++++++++++++------ 1 file changed, 40 insertions(+), 8 deletions(-) diff --git a/src/ShellCheck/Formatter/Diff.hs b/src/ShellCheck/Formatter/Diff.hs index 58fd2f4..445d9de 100644 --- a/src/ShellCheck/Formatter/Diff.hs +++ b/src/ShellCheck/Formatter/Diff.hs @@ -75,7 +75,8 @@ checkFooter didOutput color = do printErr color "Issues were detected, but none were auto-fixable. Use another format to see them." type ColorFunc = (Int -> String -> String) -data DiffDoc a = DiffDoc String [DiffRegion a] +data LFStatus = LinefeedMissing | LinefeedOk +data DiffDoc a = DiffDoc String LFStatus [DiffRegion a] data DiffRegion a = DiffRegion (Int, Int) (Int, Int) [Diff a] reportResult :: (IORef Bool) -> ColorFunc -> CheckResult -> SystemInterface IO -> IO () @@ -93,9 +94,25 @@ reportResult didOutput color result sys = do writeIORef didOutput True Left msg -> reportFailure color name msg +hasTrailingLinefeed str = + case str of + [] -> True + _ -> last str == '\n' + +coversLastLine regions = + case regions of + [] -> False + _ -> (fst $ last regions) + +-- TODO: Factor this out into a unified diff library because we're doing a lot +-- of the heavy lifting anyways. makeDiff :: String -> String -> Fix -> DiffDoc String -makeDiff name contents fix = - DiffDoc name $ findRegions . groupDiff $ computeDiff contents fix +makeDiff name contents fix = do + let hunks = groupDiff $ computeDiff contents fix + let lf = if coversLastLine hunks && not (hasTrailingLinefeed contents) + then LinefeedMissing + else LinefeedOk + DiffDoc name lf $ findRegions hunks computeDiff :: String -> Fix -> [Diff String] computeDiff contents fix = @@ -151,21 +168,36 @@ countDelta = count' 0 0 First {} -> count' (left+1) right rest Second {} -> count' left (right+1) rest -formatRegion :: ColorFunc -> DiffRegion String -> String -formatRegion color (DiffRegion left right diffs) = +formatRegion :: ColorFunc -> LFStatus -> DiffRegion String -> String +formatRegion color lf (DiffRegion left right diffs) = let header = color cyan ("@@ -" ++ (tup left) ++ " +" ++ (tup right) ++" @@") in - unlines $ header : map format diffs + unlines $ header : reverse (getStrings lf (reverse diffs)) where + noLF = "\\ No newline at end of file" + + getStrings LinefeedOk list = map format list + getStrings LinefeedMissing list@((Both _ _):_) = noLF : map format list + getStrings LinefeedMissing list@((First _):_) = noLF : map format list + getStrings LinefeedMissing (last:rest) = format last : getStrings LinefeedMissing rest + tup (a,b) = (show a) ++ "," ++ (show b) format (Both x _) = ' ':x format (First x) = color red $ '-':x format (Second x) = color green $ '+':x -formatDoc color (DiffDoc name regions) = +splitLast [] = ([], []) +splitLast x = + let (last, rest) = splitAt 1 $ reverse x + in (reverse rest, last) + +formatDoc color (DiffDoc name lf regions) = + let (most, last) = splitLast regions + in (color bold $ "--- " ++ ("a" name)) ++ "\n" ++ (color bold $ "+++ " ++ ("b" name)) ++ "\n" ++ - concatMap (formatRegion color) regions + concatMap (formatRegion color LinefeedOk) most ++ + concatMap (formatRegion color lf) last -- Create a Map from filename to Fix buildFixMap :: [Fix] -> M.Map String Fix From 3e3e4fd0cd733acf0e13b4ec5fbe577ad941a2cd Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 1 Jul 2019 23:22:09 -0700 Subject: [PATCH 160/763] Avoid defining flags for non-literal parameters --- src/ShellCheck/Analytics.hs | 1 + src/ShellCheck/AnalyzerLib.hs | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 37d97ab..b2cc179 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2087,6 +2087,7 @@ prop_checkUnused40= verifyNotTree checkUnusedAssignments "arr=(1 2); num=2; echo prop_checkUnused41= verifyNotTree checkUnusedAssignments "@test 'foo' {\ntrue\n}\n" prop_checkUnused42= verifyNotTree checkUnusedAssignments "DEFINE_string foo '' ''; echo \"${FLAGS_foo}\"" prop_checkUnused43= verifyTree checkUnusedAssignments "DEFINE_string foo '' ''" +prop_checkUnused44= verifyNotTree checkUnusedAssignments "DEFINE_string \"foo$ibar\" x y" checkUnusedAssignments params t = execWriter (mapM_ warnFor unused) where flow = variableFlow params diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 508b6ee..cf74c65 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -685,10 +685,10 @@ getModifiedVariableCommand base@(T_SimpleCommand _ _ (T_NormalWord _ (T_Literal (filter (\(x,_) -> getLiteralString x == Just "-a") (zip (args) (tail args))) -- get the FLAGS_ variable created by a shflags DEFINE_ call - getFlagVariable (n:v:_) = return (base, n, flagName n, DataString $ SourceFrom [v]) - where - flagName varName@(T_NormalWord _ _) = "FLAGS_" ++ (onlyLiteralString varName) - getFlagVariable _ = fail "Invalid flag definition" + getFlagVariable (n:v:_) = do + name <- getLiteralString v + return (base, n, "FLAGS_" ++ name, DataString $ SourceExternal) + getFlagVariable _ = Nothing getModifiedVariableCommand _ = [] From ef764b60caba1ef87f24e34dd8344894c52e65cb Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 1 Jul 2019 23:47:13 -0700 Subject: [PATCH 161/763] Fix botched variable usage --- src/ShellCheck/AnalyzerLib.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index cf74c65..581a117 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -686,7 +686,7 @@ getModifiedVariableCommand base@(T_SimpleCommand _ _ (T_NormalWord _ (T_Literal -- get the FLAGS_ variable created by a shflags DEFINE_ call getFlagVariable (n:v:_) = do - name <- getLiteralString v + name <- getLiteralString n return (base, n, "FLAGS_" ++ name, DataString $ SourceExternal) getFlagVariable _ = Nothing From bee4303c323271c8d0bd26348eb045b7412e55da Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 2 Jul 2019 20:07:05 -0700 Subject: [PATCH 162/763] Add an empty Custom.hs to simplify site-specific patching --- CHANGELOG.md | 1 + ShellCheck.cabal | 1 + src/ShellCheck/Analyzer.hs | 2 ++ src/ShellCheck/Checks/Custom.hs | 21 +++++++++++++++++++++ test/shellcheck.hs | 2 ++ 5 files changed, 27 insertions(+) create mode 100644 src/ShellCheck/Checks/Custom.hs diff --git a/CHANGELOG.md b/CHANGELOG.md index dc05c04..c9834f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ to specify search paths for sourced files. - json1 format like --format=json but treats tabs as single characters - Recognize FLAGS variables created by the shflags library. +- Site-specific changes can now be made in Custom.hs for ease of patching - SC2154: Also warn about unassigned uppercase variables (optional) - SC2252: Warn about `[ $a != x ] || [ $a != y ]`, similar to SC2055 - SC2251: Inform about ineffectual ! in front of commands diff --git a/ShellCheck.cabal b/ShellCheck.cabal index 781a082..4658dd0 100644 --- a/ShellCheck.cabal +++ b/ShellCheck.cabal @@ -74,6 +74,7 @@ library ShellCheck.AnalyzerLib ShellCheck.Checker ShellCheck.Checks.Commands + ShellCheck.Checks.Custom ShellCheck.Checks.ShellSupport ShellCheck.Data ShellCheck.Fixer diff --git a/src/ShellCheck/Analyzer.hs b/src/ShellCheck/Analyzer.hs index 01440d8..33d2ae0 100644 --- a/src/ShellCheck/Analyzer.hs +++ b/src/ShellCheck/Analyzer.hs @@ -25,6 +25,7 @@ import ShellCheck.Interface import Data.List import Data.Monoid import qualified ShellCheck.Checks.Commands +import qualified ShellCheck.Checks.Custom import qualified ShellCheck.Checks.ShellSupport @@ -41,6 +42,7 @@ analyzeScript spec = newAnalysisResult { checkers params = mconcat $ map ($ params) [ ShellCheck.Checks.Commands.checker, + ShellCheck.Checks.Custom.checker, ShellCheck.Checks.ShellSupport.checker ] diff --git a/src/ShellCheck/Checks/Custom.hs b/src/ShellCheck/Checks/Custom.hs new file mode 100644 index 0000000..76ac83c --- /dev/null +++ b/src/ShellCheck/Checks/Custom.hs @@ -0,0 +1,21 @@ +{- + This empty file is provided for ease of patching in site specific checks. + However, there are no guarantees regarding compatibility between versions. +-} + +{-# LANGUAGE TemplateHaskell #-} +module ShellCheck.Checks.Custom (checker, ShellCheck.Checks.Custom.runTests) where + +import ShellCheck.AnalyzerLib +import Test.QuickCheck + +checker :: Parameters -> Checker +checker params = Checker { + perScript = const $ return (), + perToken = const $ return () + } + +prop_CustomTestsWork = True + +return [] +runTests = $quickCheckAll diff --git a/test/shellcheck.hs b/test/shellcheck.hs index d55b140..ac84116 100644 --- a/test/shellcheck.hs +++ b/test/shellcheck.hs @@ -6,6 +6,7 @@ import qualified ShellCheck.Analytics import qualified ShellCheck.AnalyzerLib import qualified ShellCheck.Checker import qualified ShellCheck.Checks.Commands +import qualified ShellCheck.Checks.Custom import qualified ShellCheck.Checks.ShellSupport import qualified ShellCheck.Fixer import qualified ShellCheck.Formatter.Diff @@ -18,6 +19,7 @@ main = do ,ShellCheck.AnalyzerLib.runTests ,ShellCheck.Checker.runTests ,ShellCheck.Checks.Commands.runTests + ,ShellCheck.Checks.Custom.runTests ,ShellCheck.Checks.ShellSupport.runTests ,ShellCheck.Fixer.runTests ,ShellCheck.Formatter.Diff.runTests From be1f1c1ab76dcd55f06ecf99b585bba719bdbf5b Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 2 Jul 2019 20:58:08 -0700 Subject: [PATCH 163/763] Don't count 'readonly x' as a reference to x (fixes #1573) --- src/ShellCheck/Analytics.hs | 2 ++ src/ShellCheck/AnalyzerLib.hs | 4 ---- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index b2cc179..85e7afe 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2088,6 +2088,8 @@ prop_checkUnused41= verifyNotTree checkUnusedAssignments "@test 'foo' {\ntrue\n} prop_checkUnused42= verifyNotTree checkUnusedAssignments "DEFINE_string foo '' ''; echo \"${FLAGS_foo}\"" prop_checkUnused43= verifyTree checkUnusedAssignments "DEFINE_string foo '' ''" prop_checkUnused44= verifyNotTree checkUnusedAssignments "DEFINE_string \"foo$ibar\" x y" +prop_checkUnused45= verifyTree checkUnusedAssignments "readonly foo=bar" +prop_checkUnused46= verifyTree checkUnusedAssignments "readonly foo=(bar)" checkUnusedAssignments params t = execWriter (mapM_ warnFor unused) where flow = variableFlow params diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 581a117..dc0e3c4 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -546,10 +546,6 @@ getReferencedVariableCommand base@(T_SimpleCommand _ _ (T_NormalWord _ (T_Litera (not $ any (`elem` flags) ["f", "F"]) then concatMap getReference rest else [] - "readonly" -> - if any (`elem` flags) ["f", "p"] - then [] - else concatMap getReference rest "trap" -> case rest of head:_ -> map (\x -> (head, head, x)) $ getVariablesFromLiteralToken head From 4d56852b9f61267c70515b7ae648ae81ea5d119b Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 3 Jul 2019 19:49:47 -0700 Subject: [PATCH 164/763] Allow SCRIPTDIR in source directives (fixes #1617) --- shellcheck.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shellcheck.hs b/shellcheck.hs index 4ba8b70..20fb4b6 100644 --- a/shellcheck.hs +++ b/shellcheck.hs @@ -500,8 +500,8 @@ ioInterface options files = do find original original where find filename deflt = do - sources <- filterM ((allowable inputs) `andM` doesFileExist) - (map ( filename) $ map adjustPath $ sourcePathFlag ++ sourcePathAnnotation) + sources <- filterM ((allowable inputs) `andM` doesFileExist) $ + (adjustPath filename):(map ( filename) $ map adjustPath $ sourcePathFlag ++ sourcePathAnnotation) case sources of [] -> return deflt (first:_) -> return first From ba2c20a08a08a9a9d71c5fe8396bdec9479deecb Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 3 Jul 2019 20:02:14 -0700 Subject: [PATCH 165/763] Improve message for SC1067 --- src/ShellCheck/Parser.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 445d43b..b2935fd 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -2765,7 +2765,7 @@ readAssignmentWordExt lenient = try $ do variable <- readVariableName when lenient $ optional (readNormalDollar >> parseNoteAt pos ErrorC - 1067 "For indirection, use (associative) arrays or 'read \"var$n\" <<< \"value\"'") + 1067 "For indirection, use arrays, declare \"var$n=value\", or (for sh) read/eval.") indices <- many readArrayIndex hasLeftSpace <- fmap (not . null) spacing pos <- getPosition From 380221a02cc70883658cd89812c97504a424586f Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 3 Jul 2019 20:35:20 -0700 Subject: [PATCH 166/763] Recognize `read -ra foo` as arrays (fixes #1636) --- src/ShellCheck/Analytics.hs | 1 + src/ShellCheck/AnalyzerLib.hs | 11 +++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 85e7afe..c60c6ea 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -840,6 +840,7 @@ prop_checkArrayWithoutIndex6 = verifyTree checkArrayWithoutIndex "echo $PIPESTAT prop_checkArrayWithoutIndex7 = verifyTree checkArrayWithoutIndex "a=(a b); a+=c" prop_checkArrayWithoutIndex8 = verifyTree checkArrayWithoutIndex "declare -a foo; foo=bar;" prop_checkArrayWithoutIndex9 = verifyTree checkArrayWithoutIndex "read -r -a arr <<< 'foo bar'; echo \"$arr\"" +prop_checkArrayWithoutIndex10= verifyTree checkArrayWithoutIndex "read -ra arr <<< 'foo bar'; echo \"$arr\"" checkArrayWithoutIndex params _ = doVariableFlowAnalysis readF writeF defaultMap (variableFlow params) where diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index dc0e3c4..0640da2 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -676,9 +676,16 @@ getModifiedVariableCommand base@(T_SimpleCommand _ _ (T_NormalWord _ (T_Literal return (base, lastArg, name, DataArray SourceExternal) -- get all the array variables used in read, e.g. read -a arr - getReadArrayVariables args = do + getReadArrayVariables args = map (getLiteralArray . snd) - (filter (\(x,_) -> getLiteralString x == Just "-a") (zip (args) (tail args))) + (filter (isArrayFlag . fst) (zip args (tail args))) + + isArrayFlag x = fromMaybe False $ do + str <- getLiteralString x + return $ case str of + '-':'-':_ -> False + '-':str -> 'a' `elem` str + _ -> False -- get the FLAGS_ variable created by a shflags DEFINE_ call getFlagVariable (n:v:_) = do From c0d3a98fcdd952683714c313bd633ba70d34733f Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 4 Jul 2019 16:54:42 -0700 Subject: [PATCH 167/763] Add warning for chmod -r (fixes #1321) --- src/ShellCheck/Checks/Commands.hs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 851d7f2..c95720a 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -94,6 +94,7 @@ commandChecks = [ ,checkSudoRedirect ,checkSudoArgs ,checkSourceArgs + ,checkChmodDashr ] buildCommandMap :: [CommandCheck] -> Map.Map CommandName (Token -> Analysis) @@ -1042,5 +1043,16 @@ checkSourceArgs = CommandCheck (Exactly ".") f "The dot command does not support arguments in sh/dash. Set them as variables." _ -> return () +prop_checkChmodDashr1 = verify checkChmodDashr "chmod -r 0755 dir" +prop_checkChmodDashr2 = verifyNot checkChmodDashr "chmod -R 0755 dir" +prop_checkChmodDashr3 = verifyNot checkChmodDashr "chmod a-r dir" +checkChmodDashr = CommandCheck (Basename "chmod") f + where + f t = mapM_ check $ arguments t + check t = potentially $ do + flag <- getLiteralString t + guard $ flag == "-r" + return $ warn (getId t) 2253 "Use -R to recurse, or explicitly a-r to remove read permissions." + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) From 914974bd4f80fa1bddabb0a5d03bef9e8e66e9f9 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 4 Jul 2019 17:09:28 -0700 Subject: [PATCH 168/763] Don't consider `.*` a glob-like regex (fixes #1214) --- src/ShellCheck/AnalyzerLib.hs | 2 +- src/ShellCheck/Checks/Commands.hs | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 0640da2..70b781e 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -791,7 +791,7 @@ isCommandMatch token matcher = fromMaybe False $ -- False: .*foo.* isConfusedGlobRegex :: String -> Bool isConfusedGlobRegex ('*':_) = True -isConfusedGlobRegex [x,'*'] | x /= '\\' = True +isConfusedGlobRegex [x,'*'] | x `notElem` "\\." = True isConfusedGlobRegex _ = False isVariableStartChar x = x == '_' || isAsciiLower x || isAsciiUpper x diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index c95720a..f684130 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -214,6 +214,9 @@ prop_checkGrepRe17= verifyNot checkGrepRe "grep --exclude 'Foo*' file" prop_checkGrepRe18= verifyNot checkGrepRe "grep --exclude-dir 'Foo*' file" prop_checkGrepRe19= verify checkGrepRe "grep -- 'Foo*' file" prop_checkGrepRe20= verifyNot checkGrepRe "grep --fixed-strings 'Foo*' file" +prop_checkGrepRe21= verifyNot checkGrepRe "grep -o 'x*' file" +prop_checkGrepRe22= verifyNot checkGrepRe "grep --only-matching 'x*' file" +prop_checkGrepRe23= verifyNot checkGrepRe "grep '.*' file" checkGrepRe = CommandCheck (Basename "grep") check where check cmd = f cmd (arguments cmd) @@ -246,7 +249,7 @@ checkGrepRe = CommandCheck (Basename "grep") check where "Note that unlike globs, " ++ [char] ++ "* here matches '" ++ [char, char, char] ++ "' but not '" ++ wordStartingWith char ++ "'." where flags = map snd $ getAllFlags cmd - grepGlobFlags = ["fixed-strings", "F", "include", "exclude", "exclude-dir"] + grepGlobFlags = ["fixed-strings", "F", "include", "exclude", "exclude-dir", "o", "only-matching"] wordStartingWith c = head . filter ([c] `isPrefixOf`) $ candidates From 78b8e760662ee763a745b7b6d9d977ebb72274c2 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 4 Jul 2019 17:43:18 -0700 Subject: [PATCH 169/763] Also mention globbing in SC2206 (fixes #1626) --- src/ShellCheck/Analytics.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index c60c6ea..1620d22 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -3095,8 +3095,8 @@ checkSplittingInArrays params t = && not (getBracedReference (bracedString part) `elem` variablesWithoutSpaces) -> warn id 2206 $ if shellType params == Ksh - then "Quote to prevent word splitting, or split robustly with read -A or while read." - else "Quote to prevent word splitting, or split robustly with mapfile or read -a." + then "Quote to prevent word splitting/globbing, or split robustly with read -A or while read." + else "Quote to prevent word splitting/globbing, or split robustly with mapfile or read -a." _ -> return () forCommand id = From 788cf1707639283ac98b6c13400be25d9feee158 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 4 Jul 2019 19:10:14 -0700 Subject: [PATCH 170/763] Fix bad advice for SC2251 (fixes #1588) --- src/ShellCheck/Analytics.hs | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 1620d22..067a53f 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -3395,18 +3395,23 @@ checkDefaultCase _ t = pg <- wordToExactPseudoGlob pat return $ pseudoGlobIsSuperSetof pg [PGMany] -prop_checkUselessBang1 = verify checkUselessBang "! true; rest" -prop_checkUselessBang2 = verify checkUselessBang "while true; do ! true; done" -prop_checkUselessBang3 = verifyNot checkUselessBang "if ! true; then true; fi" -prop_checkUselessBang4 = verifyNot checkUselessBang "( ! true )" -prop_checkUselessBang5 = verifyNot checkUselessBang "{ ! true; }" -prop_checkUselessBang6 = verifyNot checkUselessBang "x() { ! [ x ]; }" -checkUselessBang params t = mapM_ check (getNonReturningCommands t) +prop_checkUselessBang1 = verify checkUselessBang "set -e; ! true; rest" +prop_checkUselessBang2 = verifyNot checkUselessBang "! true; rest" +prop_checkUselessBang3 = verify checkUselessBang "set -e; while true; do ! true; done" +prop_checkUselessBang4 = verifyNot checkUselessBang "set -e; if ! true; then true; fi" +prop_checkUselessBang5 = verifyNot checkUselessBang "set -e; ( ! true )" +prop_checkUselessBang6 = verify checkUselessBang "set -e; { ! true; }" +prop_checkUselessBang7 = verifyNot checkUselessBang "set -e; x() { ! [ x ]; }" +prop_checkUselessBang8 = verifyNot checkUselessBang "set -e; if { ! true; }; then true; fi" +prop_checkUselessBang9 = verifyNot checkUselessBang "set -e; while ! true; do true; done" +checkUselessBang params t = when (hasSetE params) $ mapM_ check (getNonReturningCommands t) where check t = case t of - T_Banged id _ -> - info id 2251 "This ! is not on a condition and skips errexit. Use { ! ...; } to errexit, or verify usage." + T_Banged id cmd | not $ isCondition (getPath (parentMap params) t) -> + addComment $ makeCommentWithFix InfoC id 2251 + "This ! is not on a condition and skips errexit. Use `&& exit 1` instead, or make sure $? is checked." + (fixWith [replaceStart id params 1 "", replaceEnd (getId cmd) params 0 " && exit 1"]) _ -> return () -- Get all the subcommands that aren't likely to be the return value @@ -3414,7 +3419,7 @@ checkUselessBang params t = mapM_ check (getNonReturningCommands t) getNonReturningCommands t = case t of T_Script _ _ list -> dropLast list - T_BraceGroup _ list -> dropLast list + T_BraceGroup _ list -> if isFunctionBody t then dropLast list else list T_Subshell _ list -> dropLast list T_WhileExpression _ conds cmds -> dropLast conds ++ cmds T_UntilExpression _ conds cmds -> dropLast conds ++ cmds @@ -3425,6 +3430,11 @@ checkUselessBang params t = mapM_ check (getNonReturningCommands t) concatMap (dropLast . fst) conds ++ concatMap snd conds ++ elses _ -> [] + isFunctionBody t = + case getPath (parentMap params) t of + _:T_Function {}:_-> True + _ -> False + dropLast t = case t of [_] -> [] From e280116ef0ea23c078af1e41c70838e98d0c4828 Mon Sep 17 00:00:00 2001 From: shak-mar Date: Fri, 12 Jul 2019 16:48:27 +0200 Subject: [PATCH 171/763] Fix syntax and indentation in shellcheck.1.md Out of interest, I ran the command pandoc -s -f markdown-smart -t man shellcheck.1.md -o shellcheck.1 locally, but that produces warnings (previous to this commit). Checking the generated manpage, I found the diff to be rendered very badly. (Broken at terminal width like a normal paragraph). This commit fixes the problem. --- shellcheck.1.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/shellcheck.1.md b/shellcheck.1.md index 77fa79a..da66a69 100644 --- a/shellcheck.1.md +++ b/shellcheck.1.md @@ -157,16 +157,16 @@ not warn at all, as `ksh` supports decimals in arithmetic contexts. : Auto-fixes in unified diff format. Can be piped to `git apply` or `patch -p1` to automatically apply fixes. - --- a/test.sh - +++ b/test.sh - @@ -2,6 +2,6 @@ - ## Example of a broken script. - for f in $(ls *.m3u) - do - - grep -qi hq.*mp3 $f \ - + grep -qi hq.*mp3 "$f" \ - && echo -e 'Playlist $f contains a HQ file in mp3 format' - done + --- a/test.sh + +++ b/test.sh + @@ -2,6 +2,6 @@ + ## Example of a broken script. + for f in $(ls *.m3u) + do + - grep -qi hq.*mp3 $f \ + + grep -qi hq.*mp3 "$f" \ + && echo -e 'Playlist $f contains a HQ file in mp3 format' + done **json1** From 023ae5dfdabf5c347fda76694d96c9ef50f8068b Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 20 Jul 2019 15:10:41 -0700 Subject: [PATCH 172/763] Don't warn about printf '%()T' without corresponding argument --- CHANGELOG.md | 3 + src/ShellCheck/Checks/Commands.hs | 91 +++++++++++++++++++++---------- 2 files changed, 64 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c9834f7..45c9a0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,9 @@ extension will be used to infer the shell type when present. - Disabling SC2120 on a function now disables SC2119 on call sites +### Fixed +- SC2183 no longer warns about missing printf args for `%()T` + ## v0.6.0 - 2018-12-02 ### Added - Command line option --severity/-S for filtering by minimum severity diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index f684130..c6346a9 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -538,52 +538,83 @@ prop_checkPrintfVar15= verifyNot checkPrintfVar "printf '%*s\\n' 1 2" prop_checkPrintfVar16= verifyNot checkPrintfVar "printf $'string'" prop_checkPrintfVar17= verify checkPrintfVar "printf '%-*s\\n' 1" prop_checkPrintfVar18= verifyNot checkPrintfVar "printf '%-*s\\n' 1 2" +prop_checkPrintfVar19= verifyNot checkPrintfVar "printf '%(%s)T'" +prop_checkPrintfVar20= verifyNot checkPrintfVar "printf '%d %(%s)T' 42" +prop_checkPrintfVar21= verify checkPrintfVar "printf '%d %(%s)T'" checkPrintfVar = CommandCheck (Exactly "printf") (f . arguments) where f (doubledash:rest) | getLiteralString doubledash == Just "--" = f rest f (dashv:var:rest) | getLiteralString dashv == Just "-v" = f rest f (format:params) = check format params f _ = return () - countFormats string = - case string of - '%':'%':rest -> countFormats rest - '%':'(':rest -> 1 + countFormats (dropWhile (/= ')') rest) - '%':rest -> regexBasedCountFormats rest + countFormats (dropWhile (/= '%') rest) - _:rest -> countFormats rest - [] -> 0 - - regexBasedCountFormats rest = - maybe 1 (foldl (\acc group -> acc + (if group == "*" then 1 else 0)) 1) (matchRegex re rest) - where - -- constructed based on specifications in "man printf" - re = mkRegex "#?-?\\+? ?0?(\\*|\\d*).?(\\d*|\\*)[diouxXfFeEgGaAcsb]" - -- \____ _____/\___ ____/ \____ ____/\________ ________/ - -- V V V V - -- flags field width precision format character - -- field width and precision can be specified with a '*' instead of a digit, - -- in which case printf will accept one more argument for each '*' used check format more = do fromMaybe (return ()) $ do string <- getLiteralString format - let vars = countFormats string - - return $ do - when (vars == 0 && more /= []) $ - err (getId format) 2182 - "This printf format string has no variables. Other arguments are ignored." - - when (vars > 0 - && ((length more) `mod` vars /= 0 || null more) - && all (not . mayBecomeMultipleArgs) more) $ - warn (getId format) 2183 $ - "This format string has " ++ show vars ++ " variables, but is passed " ++ show (length more) ++ " arguments." + let formats = getPrintfFormats string + let formatCount = length formats + let argCount = length more + return $ + case () of + () | argCount == 0 && formatCount == 0 -> + return () -- This is fine + () | formatCount == 0 && argCount > 0 -> + err (getId format) 2182 + "This printf format string has no variables. Other arguments are ignored." + () | any mayBecomeMultipleArgs more -> + return () -- We don't know so trust the user + () | argCount < formatCount && onlyTrailingTs formats argCount -> + return () -- Allow trailing %()Ts since they use the current time + () | argCount > 0 && argCount `mod` formatCount == 0 -> + return () -- Great: a suitable number of arguments + () -> + warn (getId format) 2183 $ + "This format string has " ++ show formatCount ++ " variables, but is passed " ++ show argCount ++ " arguments." unless ('%' `elem` concat (oversimplify format) || isLiteral format) $ info (getId format) 2059 "Don't use variables in the printf format string. Use printf \"..%s..\" \"$foo\"." + where + onlyTrailingTs format argCount = + all (== 'T') $ drop argCount format +prop_checkGetPrintfFormats1 = getPrintfFormats "%s" == "s" +prop_checkGetPrintfFormats2 = getPrintfFormats "%0*s" == "*s" +prop_checkGetPrintfFormats3 = getPrintfFormats "%(%s)T" == "T" +prop_checkGetPrintfFormats4 = getPrintfFormats "%d%%%(%s)T" == "dT" +prop_checkGetPrintfFormats5 = getPrintfFormats "%bPassed: %d, %bFailed: %d%b, Skipped: %d, %bErrored: %d%b\\n" == "bdbdbdbdb" +getPrintfFormats = getFormats + where + -- Get the arguments in the string as a string of type characters, + -- e.g. "Hello %s" -> "s" and "%(%s)T %0*d\n" -> "T*d" + getFormats :: String -> String + getFormats string = + case string of + '%':'%':rest -> getFormats rest + '%':'(':rest -> + case dropWhile (/= ')') rest of + ')':c:trailing -> c : getFormats trailing + _ -> "" + '%':rest -> regexBasedGetFormats rest + _:rest -> getFormats rest + [] -> "" + + regexBasedGetFormats rest = + case matchRegex re rest of + Just [width, precision, typ, rest] -> + (if width == "*" then "*" else "") ++ + (if precision == "*" then "*" else "") ++ + typ ++ getFormats rest + Nothing -> take 1 rest ++ getFormats rest + where + -- constructed based on specifications in "man printf" + re = mkRegex "#?-?\\+? ?0?(\\*|\\d*)\\.?(\\d*|\\*)([diouxXfFeEgGaAcsbq])(.*)" + -- \____ _____/\___ ____/ \____ ____/\_________ _________/ \ / + -- V V V V V + -- flags field width precision format character rest + -- field width and precision can be specified with a '*' instead of a digit, + -- in which case printf will accept one more argument for each '*' used prop_checkUuoeCmd1 = verify checkUuoeCmd "echo $(date)" From 38bb156a1cda803b823196ad6015a0b3b70678b4 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 21 Jul 2019 21:22:16 -0700 Subject: [PATCH 173/763] Warn about $_ in POSIX sh (fixes #1647) --- src/ShellCheck/Checks/ShellSupport.hs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 4a86891..83d23fb 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -175,6 +175,8 @@ prop_checkBashisms91 = verify checkBashisms "#!/bin/sh\nwait -n" prop_checkBashisms92 = verify checkBashisms "#!/bin/sh\necho $((16#FF))" prop_checkBashisms93 = verify checkBashisms "#!/bin/sh\necho $(( 10#$(date +%m) ))" prop_checkBashisms94 = verify checkBashisms "#!/bin/sh\n[ -v var ]" +prop_checkBashisms95 = verify checkBashisms "#!/bin/sh\necho $_" +prop_checkBashisms96 = verifyNot checkBashisms "#!/bin/dash\necho $_" checkBashisms = ForShell [Sh, Dash] $ \t -> do params <- ask kludge params t @@ -408,10 +410,11 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do ] bashVars = [ "OSTYPE", "MACHTYPE", "HOSTTYPE", "HOSTNAME", - "DIRSTACK", "EUID", "UID", "SHLVL", "PIPESTATUS", "SHELLOPTS" + "DIRSTACK", "EUID", "UID", "SHLVL", "PIPESTATUS", "SHELLOPTS", + "_" ] bashDynamicVars = [ "RANDOM", "SECONDS" ] - dashVars = [ ] + dashVars = [ "_" ] isBashVariable var = (var `elem` bashDynamicVars || var `elem` bashVars && not (isAssigned var)) From 7a1fb2523df649ab727057f36473adb9228ef806 Mon Sep 17 00:00:00 2001 From: Luizm Date: Wed, 29 May 2019 12:10:06 -0300 Subject: [PATCH 174/763] Add support to compiling a binary for macOS --- .compile_binaries | 71 ++++++++++++++++++++++++++++++++++++++++++++ .prepare_deploy | 13 +++++++- .travis.yml | 75 +++++++++++++++-------------------------------- README.md | 5 ++-- 4 files changed, 109 insertions(+), 55 deletions(-) create mode 100755 .compile_binaries diff --git a/.compile_binaries b/.compile_binaries new file mode 100755 index 0000000..1aa18ad --- /dev/null +++ b/.compile_binaries @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# This script compile shellcheck binaries +set -ex + +# Remove all tests to reduce binary size +./striptests + +mkdir -p deploy + +_cleanup(){ + rm -rf dist shellcheck || true +} + +if [ "$TRAVIS_OS_NAME" = 'linux' ] +then + # Linux Docker image + name="$DOCKER_BASE" + DOCKER_BUILDS="$DOCKER_BUILDS $name" + docker build -t "$name:current" . + docker run "$name:current" --version + printf '%s\n' "#!/bin/sh" "echo 'hello world'" > myscript + docker run -v "$PWD:/mnt" "$name:current" myscript + + # Copy static executable from docker image + id=$(docker create "$name:current") + docker cp "$id:/bin/shellcheck" "shellcheck" + docker rm "$id" + ls -l shellcheck + ./shellcheck myscript + for tag in $TAGS + do + cp "shellcheck" "deploy/shellcheck-$tag.linux-x86_64"; + done + + # Linux Alpine based Docker image + name="$DOCKER_BASE-alpine" + DOCKER_BUILDS="$DOCKER_BUILDS $name" + sed -e '/DELETE-MARKER/,$d' Dockerfile > Dockerfile.alpine + docker build -f Dockerfile.alpine -t "$name:current" . + docker run "$name:current" sh -c 'shellcheck --version' + + # Linux armv6hf static executable + docker run -v "$PWD:/mnt" koalaman/armv6hf-builder -c 'compile-shellcheck' + for tag in $TAGS + do + cp "shellcheck" "deploy/shellcheck-$tag.linux-armv6hf"; + done + _cleanup + + # Windows .exe + docker run --user="$UID" -v "$PWD:/appdata" koalaman/winghc cuib + for tag in $TAGS + do + cp "dist/build/ShellCheck/shellcheck.exe" "deploy/shellcheck-$tag.exe"; + done + _cleanup +fi + +if [ "$TRAVIS_OS_NAME" = 'osx' ]; +then + # Darwin x86_64 static executable + sudo ln -s /usr/local/bin/gsha512sum /usr/local/bin/sha512sum + brew install cabal-install pandoc + cabal update + cabal new-build shellcheck + for tag in $TAGS + do + cp "$HOME/.cabal/dist/build/shellcheck/shellcheck" "deploy/shellcheck-$tag.darwin-x86_64"; + done + _cleanup +fi diff --git a/.prepare_deploy b/.prepare_deploy index dcf0346..490bf62 100755 --- a/.prepare_deploy +++ b/.prepare_deploy @@ -43,8 +43,19 @@ do rm "shellcheck" done +if [ "$TRAVIS_OS_NAME" = 'osx' ]; +then + brew install gnu-tar + for file in *.darwin-x86_64 + do + base="${file%.*}" + cp "$file" "shellcheck" + gtar -cJf "$base.darwin.x86_64.tar.xz" --transform="s:^:$base/:" README.txt LICENSE.txt shellcheck + rm "shellcheck" + done +fi + for file in ./* do sha512sum "$file" > "$file.sha512sum" done - diff --git a/.travis.yml b/.travis.yml index 8a86307..25ebbeb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,67 +5,40 @@ language: sh services: - docker -before_install: - - DOCKER_BASE="$DOCKER_USERNAME/shellcheck" - - DOCKER_BUILDS="" - - TAGS="" - - test "$TRAVIS_BRANCH" = master && TAGS="$TAGS latest" || true - - test -n "$TRAVIS_TAG" && TAGS="$TAGS stable $TRAVIS_TAG" || true - - echo "Tags are $TAGS" +os: + - linux + - osx + +before_install: | + DOCKER_BASE="$DOCKER_USERNAME/shellcheck" + DOCKER_BUILDS="" + TAGS="" + test "$TRAVIS_BRANCH" = master && TAGS="$TAGS latest" || true + test -n "$TRAVIS_TAG" && TAGS="$TAGS stable $TRAVIS_TAG" || true + echo "Tags are $TAGS" script: - - mkdir deploy - # Remove all tests to reduce binary size - - ./striptests - # Linux Docker image - - name="$DOCKER_BASE" - - DOCKER_BUILDS="$DOCKER_BUILDS $name" - - docker build -t "$name:current" . - - docker run "$name:current" --version - - printf '%s\n' "#!/bin/sh" "echo 'hello world'" > myscript - - docker run -v "$PWD:/mnt" "$name:current" myscript - # Copy static executable from docker image - - id=$(docker create "$name:current") - - docker cp "$id:/bin/shellcheck" "shellcheck" - - docker rm "$id" - - ls -l shellcheck - - ./shellcheck myscript - - for tag in $TAGS; do cp "shellcheck" "deploy/shellcheck-$tag.linux-x86_64"; done - # Linux Alpine based Docker image - - name="$DOCKER_BASE-alpine" - - DOCKER_BUILDS="$DOCKER_BUILDS $name" - - sed -e '/DELETE-MARKER/,$d' Dockerfile > Dockerfile.alpine - - docker build -f Dockerfile.alpine -t "$name:current" . - - docker run "$name:current" sh -c 'shellcheck --version' - # Linux armv6hf static executable - - docker run -v "$PWD:/mnt" koalaman/armv6hf-builder -c 'compile-shellcheck' - - for tag in $TAGS; do cp "shellcheck" "deploy/shellcheck-$tag.linux-armv6hf"; done - - rm -f shellcheck || true - # Windows .exe - - docker run --user="$UID" -v "$PWD:/appdata" koalaman/winghc cuib - - for tag in $TAGS; do cp "dist/build/ShellCheck/shellcheck.exe" "deploy/shellcheck-$tag.exe"; done - - rm -rf dist shellcheck || true - # Misc packaging + - ./.compile_binaries - ./.prepare_deploy -after_success: - - docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD" - - for repo in $DOCKER_BUILDS; - do - for tag in $TAGS; - do +after_success: | + if [ "$TRAVIS_OS_NAME" = "linux" ]; then + docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD" + for repo in $DOCKER_BUILDS; do + for tag in $TAGS; do echo "Deploying $repo:current as $repo:$tag..."; docker tag "$repo:current" "$repo:$tag" || exit 1; docker push "$repo:$tag" || exit 1; done; done; + fi -after_failure: - - id - - pwd - - df -h - - find . -name '*.log' -type f -exec grep "" /dev/null {} + - - find . -ls +after_failure: | + id + pwd + df -h + find . -name '*.log' -type f -exec grep "" /dev/null {} + + find . -ls deploy: provider: gcs diff --git a/README.md b/README.md index 6b789cd..4c9e9ae 100644 --- a/README.md +++ b/README.md @@ -257,9 +257,7 @@ ShellCheck is built and packaged using Cabal. Install the package `cabal-install On MacOS (OS X), you can do a fast install of Cabal using brew, which takes a couple of minutes instead of more than 30 minutes if you try to compile it from source. - brew install cask - brew cask install haskell-platform - cabal install cabal-install + $ brew install cabal-install On MacPorts, the package is instead called `hs-cabal-install`, while native Windows users should install the latest version of the Haskell platform from @@ -512,3 +510,4 @@ Happy ShellChecking! * The wiki has [long form descriptions](https://github.com/koalaman/shellcheck/wiki/Checks) for each warning, e.g. [SC2221](https://github.com/koalaman/shellcheck/wiki/SC2221). * ShellCheck does not attempt to enforce any kind of formatting or indenting style, so also check out [shfmt](https://github.com/mvdan/sh)! + From 25b5b77240c94d3a28ae9aa7bcef36afb19b00c2 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 24 Jul 2019 19:02:10 -0700 Subject: [PATCH 175/763] Add automated linux-aarch64 build --- .prepare_deploy | 8 ++++++++ .travis.yml | 7 +++++++ 2 files changed, 15 insertions(+) diff --git a/.prepare_deploy b/.prepare_deploy index dcf0346..fd11fb1 100755 --- a/.prepare_deploy +++ b/.prepare_deploy @@ -35,6 +35,14 @@ do rm "shellcheck" done +for file in *.linux-aarch64 +do + base="${file%.*}" + cp "$file" "shellcheck" + tar -cJf "$base.linux.aarch64.tar.xz" --transform="s:^:$base/:" README.txt LICENSE.txt shellcheck + rm "shellcheck" +done + for file in *.linux-armv6hf do base="${file%.*}" diff --git a/.travis.yml b/.travis.yml index 8a86307..e921a5f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,6 +17,8 @@ script: - mkdir deploy # Remove all tests to reduce binary size - ./striptests + # Start fetching the aarch64 image since it's a multi-GB beast + - docker pull koalaman/aarch64-builder >> aarch64pull.log 2>&1 & # Linux Docker image - name="$DOCKER_BASE" - DOCKER_BUILDS="$DOCKER_BUILDS $name" @@ -37,6 +39,11 @@ script: - sed -e '/DELETE-MARKER/,$d' Dockerfile > Dockerfile.alpine - docker build -f Dockerfile.alpine -t "$name:current" . - docker run "$name:current" sh -c 'shellcheck --version' + # Linux aarch64 static executable + - wait + - docker run -v "$PWD:/mnt" koalaman/aarch64-builder 'buildsc' + - for tag in $TAGS; do cp "shellcheck" "deploy/shellcheck-$tag.linux-aarch64"; done + - rm -f shellcheck || true # Linux armv6hf static executable - docker run -v "$PWD:/mnt" koalaman/armv6hf-builder -c 'compile-shellcheck' - for tag in $TAGS; do cp "shellcheck" "deploy/shellcheck-$tag.linux-armv6hf"; done From 0eaef95db85a0a47733c52e677eab8576d98adaa Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 24 Jul 2019 21:16:14 -0700 Subject: [PATCH 176/763] THIS COMMIT WILL BE FORCE PUSHED AWAY (Help I'm not good with computers) --- .compile_binaries | 78 ++++++++++++++++++++++++++++++++++++++++++++ .prepare_deploy | 13 +++++++- .travis.yml | 82 ++++++++++++++--------------------------------- README.md | 5 ++- 4 files changed, 116 insertions(+), 62 deletions(-) create mode 100755 .compile_binaries diff --git a/.compile_binaries b/.compile_binaries new file mode 100755 index 0000000..8fa645c --- /dev/null +++ b/.compile_binaries @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# This script compile shellcheck binaries +set -ex + +# Remove all tests to reduce binary size +./striptests + +mkdir -p deploy + +_cleanup(){ + rm -rf dist shellcheck || true +} + +if [ "$TRAVIS_OS_NAME" = 'linux' ] +then + # Linux Docker image + name="$DOCKER_BASE" + DOCKER_BUILDS="$DOCKER_BUILDS $name" + docker build -t "$name:current" . + docker run "$name:current" --version + printf '%s\n' "#!/bin/sh" "echo 'hello world'" > myscript + docker run -v "$PWD:/mnt" "$name:current" myscript + + # Copy static executable from docker image + id=$(docker create "$name:current") + docker cp "$id:/bin/shellcheck" "shellcheck" + docker rm "$id" + ls -l shellcheck + ./shellcheck myscript + for tag in $TAGS + do + cp "shellcheck" "deploy/shellcheck-$tag.linux-x86_64"; + done + + # Linux Alpine based Docker image + name="$DOCKER_BASE-alpine" + DOCKER_BUILDS="$DOCKER_BUILDS $name" + sed -e '/DELETE-MARKER/,$d' Dockerfile > Dockerfile.alpine + docker build -f Dockerfile.alpine -t "$name:current" . + docker run "$name:current" sh -c 'shellcheck --version' + + # Linux aarch64 static executable + docker run -v "$PWD:/mnt" koalaman/aarch64-builder 'buildsc' + for tag in $TAGS + do + cp "shellcheck" "deploy/shellcheck-$tag.linux-aarch64" + done + + # Linux armv6hf static executable + docker run -v "$PWD:/mnt" koalaman/armv6hf-builder -c 'compile-shellcheck' + for tag in $TAGS + do + cp "shellcheck" "deploy/shellcheck-$tag.linux-armv6hf"; + done + _cleanup + + # Windows .exe + docker run --user="$UID" -v "$PWD:/appdata" koalaman/winghc cuib + for tag in $TAGS + do + cp "dist/build/ShellCheck/shellcheck.exe" "deploy/shellcheck-$tag.exe"; + done + _cleanup +fi + +if [ "$TRAVIS_OS_NAME" = 'osx' ]; +then + # Darwin x86_64 static executable + sudo ln -s /usr/local/bin/gsha512sum /usr/local/bin/sha512sum + brew install cabal-install pandoc + cabal update + cabal new-build shellcheck + for tag in $TAGS + do + cp "$HOME/.cabal/dist/build/shellcheck/shellcheck" "deploy/shellcheck-$tag.darwin-x86_64"; + done + _cleanup +fi diff --git a/.prepare_deploy b/.prepare_deploy index fd11fb1..030c3fa 100755 --- a/.prepare_deploy +++ b/.prepare_deploy @@ -51,8 +51,19 @@ do rm "shellcheck" done +if [ "$TRAVIS_OS_NAME" = 'osx' ]; +then + brew install gnu-tar + for file in *.darwin-x86_64 + do + base="${file%.*}" + cp "$file" "shellcheck" + gtar -cJf "$base.darwin.x86_64.tar.xz" --transform="s:^:$base/:" README.txt LICENSE.txt shellcheck + rm "shellcheck" + done +fi + for file in ./* do sha512sum "$file" > "$file.sha512sum" done - diff --git a/.travis.yml b/.travis.yml index e921a5f..25ebbeb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,74 +5,40 @@ language: sh services: - docker -before_install: - - DOCKER_BASE="$DOCKER_USERNAME/shellcheck" - - DOCKER_BUILDS="" - - TAGS="" - - test "$TRAVIS_BRANCH" = master && TAGS="$TAGS latest" || true - - test -n "$TRAVIS_TAG" && TAGS="$TAGS stable $TRAVIS_TAG" || true - - echo "Tags are $TAGS" +os: + - linux + - osx + +before_install: | + DOCKER_BASE="$DOCKER_USERNAME/shellcheck" + DOCKER_BUILDS="" + TAGS="" + test "$TRAVIS_BRANCH" = master && TAGS="$TAGS latest" || true + test -n "$TRAVIS_TAG" && TAGS="$TAGS stable $TRAVIS_TAG" || true + echo "Tags are $TAGS" script: - - mkdir deploy - # Remove all tests to reduce binary size - - ./striptests - # Start fetching the aarch64 image since it's a multi-GB beast - - docker pull koalaman/aarch64-builder >> aarch64pull.log 2>&1 & - # Linux Docker image - - name="$DOCKER_BASE" - - DOCKER_BUILDS="$DOCKER_BUILDS $name" - - docker build -t "$name:current" . - - docker run "$name:current" --version - - printf '%s\n' "#!/bin/sh" "echo 'hello world'" > myscript - - docker run -v "$PWD:/mnt" "$name:current" myscript - # Copy static executable from docker image - - id=$(docker create "$name:current") - - docker cp "$id:/bin/shellcheck" "shellcheck" - - docker rm "$id" - - ls -l shellcheck - - ./shellcheck myscript - - for tag in $TAGS; do cp "shellcheck" "deploy/shellcheck-$tag.linux-x86_64"; done - # Linux Alpine based Docker image - - name="$DOCKER_BASE-alpine" - - DOCKER_BUILDS="$DOCKER_BUILDS $name" - - sed -e '/DELETE-MARKER/,$d' Dockerfile > Dockerfile.alpine - - docker build -f Dockerfile.alpine -t "$name:current" . - - docker run "$name:current" sh -c 'shellcheck --version' - # Linux aarch64 static executable - - wait - - docker run -v "$PWD:/mnt" koalaman/aarch64-builder 'buildsc' - - for tag in $TAGS; do cp "shellcheck" "deploy/shellcheck-$tag.linux-aarch64"; done - - rm -f shellcheck || true - # Linux armv6hf static executable - - docker run -v "$PWD:/mnt" koalaman/armv6hf-builder -c 'compile-shellcheck' - - for tag in $TAGS; do cp "shellcheck" "deploy/shellcheck-$tag.linux-armv6hf"; done - - rm -f shellcheck || true - # Windows .exe - - docker run --user="$UID" -v "$PWD:/appdata" koalaman/winghc cuib - - for tag in $TAGS; do cp "dist/build/ShellCheck/shellcheck.exe" "deploy/shellcheck-$tag.exe"; done - - rm -rf dist shellcheck || true - # Misc packaging + - ./.compile_binaries - ./.prepare_deploy -after_success: - - docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD" - - for repo in $DOCKER_BUILDS; - do - for tag in $TAGS; - do +after_success: | + if [ "$TRAVIS_OS_NAME" = "linux" ]; then + docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD" + for repo in $DOCKER_BUILDS; do + for tag in $TAGS; do echo "Deploying $repo:current as $repo:$tag..."; docker tag "$repo:current" "$repo:$tag" || exit 1; docker push "$repo:$tag" || exit 1; done; done; + fi -after_failure: - - id - - pwd - - df -h - - find . -name '*.log' -type f -exec grep "" /dev/null {} + - - find . -ls +after_failure: | + id + pwd + df -h + find . -name '*.log' -type f -exec grep "" /dev/null {} + + find . -ls deploy: provider: gcs diff --git a/README.md b/README.md index 3baca68..4c9e9ae 100644 --- a/README.md +++ b/README.md @@ -257,9 +257,7 @@ ShellCheck is built and packaged using Cabal. Install the package `cabal-install On MacOS (OS X), you can do a fast install of Cabal using brew, which takes a couple of minutes instead of more than 30 minutes if you try to compile it from source. - brew install cask - brew cask install haskell-for-mac - cabal install cabal-install + $ brew install cabal-install On MacPorts, the package is instead called `hs-cabal-install`, while native Windows users should install the latest version of the Haskell platform from @@ -512,3 +510,4 @@ Happy ShellChecking! * The wiki has [long form descriptions](https://github.com/koalaman/shellcheck/wiki/Checks) for each warning, e.g. [SC2221](https://github.com/koalaman/shellcheck/wiki/SC2221). * ShellCheck does not attempt to enforce any kind of formatting or indenting style, so also check out [shfmt](https://github.com/mvdan/sh)! + From f9c8a255bee3a65fe895e31c1d3c4b1ae1147f39 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 24 Jul 2019 22:05:30 -0700 Subject: [PATCH 177/763] Set up Travis build matrix --- .compile_binaries | 38 +++++++++++++++++++++----------------- .prepare_deploy | 18 +++++++----------- .travis.yml | 23 ++++++++++++++++++----- 3 files changed, 46 insertions(+), 33 deletions(-) diff --git a/.compile_binaries b/.compile_binaries index 8fa645c..cf6c848 100755 --- a/.compile_binaries +++ b/.compile_binaries @@ -1,18 +1,10 @@ -#!/usr/bin/env bash -# This script compile shellcheck binaries -set -ex - -# Remove all tests to reduce binary size -./striptests - -mkdir -p deploy +#!/bin/bash _cleanup(){ rm -rf dist shellcheck || true } -if [ "$TRAVIS_OS_NAME" = 'linux' ] -then +build_linux() { # Linux Docker image name="$DOCKER_BASE" DOCKER_BUILDS="$DOCKER_BUILDS $name" @@ -38,14 +30,20 @@ then sed -e '/DELETE-MARKER/,$d' Dockerfile > Dockerfile.alpine docker build -f Dockerfile.alpine -t "$name:current" . docker run "$name:current" sh -c 'shellcheck --version' + _cleanup +} +build_aarch64() { # Linux aarch64 static executable docker run -v "$PWD:/mnt" koalaman/aarch64-builder 'buildsc' for tag in $TAGS do cp "shellcheck" "deploy/shellcheck-$tag.linux-aarch64" done +} + +build_armv6hf() { # Linux armv6hf static executable docker run -v "$PWD:/mnt" koalaman/armv6hf-builder -c 'compile-shellcheck' for tag in $TAGS @@ -53,7 +51,9 @@ then cp "shellcheck" "deploy/shellcheck-$tag.linux-armv6hf"; done _cleanup +} +build_windows() { # Windows .exe docker run --user="$UID" -v "$PWD:/appdata" koalaman/winghc cuib for tag in $TAGS @@ -61,18 +61,22 @@ then cp "dist/build/ShellCheck/shellcheck.exe" "deploy/shellcheck-$tag.exe"; done _cleanup -fi +} -if [ "$TRAVIS_OS_NAME" = 'osx' ]; -then +build_osx() { # Darwin x86_64 static executable + brew install cabal-install pandoc gnu-tar sudo ln -s /usr/local/bin/gsha512sum /usr/local/bin/sha512sum - brew install cabal-install pandoc + sudo ln -s /usr/local/bin/gtar /usr/local/bin/tar + export PATH="/usr/local/bin:$PATH" + cabal update - cabal new-build shellcheck + cabal install --dependencies-only + cabal build shellcheck for tag in $TAGS do - cp "$HOME/.cabal/dist/build/shellcheck/shellcheck" "deploy/shellcheck-$tag.darwin-x86_64"; + cp "dist/build/shellcheck/shellcheck" "deploy/shellcheck-$tag.darwin-x86_64"; done _cleanup -fi +} + diff --git a/.prepare_deploy b/.prepare_deploy index 030c3fa..5e4ffaf 100755 --- a/.prepare_deploy +++ b/.prepare_deploy @@ -51,17 +51,13 @@ do rm "shellcheck" done -if [ "$TRAVIS_OS_NAME" = 'osx' ]; -then - brew install gnu-tar - for file in *.darwin-x86_64 - do - base="${file%.*}" - cp "$file" "shellcheck" - gtar -cJf "$base.darwin.x86_64.tar.xz" --transform="s:^:$base/:" README.txt LICENSE.txt shellcheck - rm "shellcheck" - done -fi +for file in *.darwin-x86_64 +do + base="${file%.*}" + cp "$file" "shellcheck" + tar -cJf "$base.darwin.x86_64.tar.xz" --transform="s:^:$base/:" README.txt LICENSE.txt shellcheck + rm "shellcheck" +done for file in ./* do diff --git a/.travis.yml b/.travis.yml index 25ebbeb..81fcb2e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,4 @@ + sudo: required language: sh @@ -5,9 +6,18 @@ language: sh services: - docker -os: - - linux - - osx +matrix: + include: + - os: linux + env: BUILD=linux + - os: linux + env: BUILD=windows + - os: linux + env: BUILD=armv6hf + - os: linux + env: BUILD=aarch64 + - os: osx + env: BUILD=osx before_install: | DOCKER_BASE="$DOCKER_USERNAME/shellcheck" @@ -18,11 +28,14 @@ before_install: | echo "Tags are $TAGS" script: - - ./.compile_binaries + - mkdir -p deploy + - source ./.compile_binaries + - ./striptests + - set -x; build_"$BUILD"; set +x; - ./.prepare_deploy after_success: | - if [ "$TRAVIS_OS_NAME" = "linux" ]; then + if [ "$BUILD" = "linux" ]; then docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD" for repo in $DOCKER_BUILDS; do for tag in $TAGS; do From e4cbf59fda4111277b11fcd8957ca8302437d636 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 28 Jul 2019 17:10:20 -0700 Subject: [PATCH 178/763] Update distrotest with new image names --- test/distrotest | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/distrotest b/test/distrotest index 5024054..d706185 100755 --- a/test/distrotest +++ b/test/distrotest @@ -61,11 +61,11 @@ done << EOF debian:stable apt-get update && apt-get install -y cabal-install debian:testing apt-get update && apt-get install -y cabal-install ubuntu:latest apt-get update && apt-get install -y cabal-install -opensuse:latest zypper install -y cabal-install ghc +opensuse/leap:latest zypper install -y cabal-install ghc -# Older Ubuntu versions we want to support -ubuntu:18.04 apt-get update && apt-get install -y cabal-install -ubuntu:17.10 apt-get update && apt-get install -y cabal-install +# Other Ubuntu versions we want to support +ubuntu:19.04 apt-get update && apt-get install -y cabal-install +ubuntu:18.10 apt-get update && apt-get install -y cabal-install # Misc Haskell including current and latest Stack build ubuntu:18.10 set -e; apt-get update && apt-get install -y curl && curl -sSL https://get.haskellstack.org/ | sh -s - -f && cd /mnt && exec test/stacktest @@ -74,7 +74,7 @@ haskell:latest true # Known to currently fail centos:latest yum install -y epel-release && yum install -y cabal-install fedora:latest dnf install -y cabal-install -base/archlinux:latest pacman -S -y --noconfirm cabal-install ghc-static base-devel +archlinux/base:latest pacman -S -y --noconfirm cabal-install ghc-static base-devel EOF exit "$final" From 2053ac8882631541e6e859584e4f6a6e44aecd34 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 28 Jul 2019 17:10:59 -0700 Subject: [PATCH 179/763] Add a release checklist script --- test/check_release | 75 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100755 test/check_release diff --git a/test/check_release b/test/check_release new file mode 100755 index 0000000..5e73d9e --- /dev/null +++ b/test/check_release @@ -0,0 +1,75 @@ +#!/usr/bin/env bash + +failed=0 +fail() { + echo "$(tput setaf 1)$*$(tput sgr0)" + failed=1 +} + +if git diff | grep -q "" +then + fail "There are uncommited changes" +fi + +current=$(git tag --points-at) +if [[ -z "$current" ]] +then + fail "No git tag on the current commit" + echo "Create one with: git tag -a v0.0.0" +fi + +if [[ "$current" != v* ]] +then + fail "Bad tag format: expected v0.0.0" +fi + +if [[ "$(git cat-file -t "$current")" != "tag" ]] +then + fail "Current tag is not annotated (required for Snap)." +fi + +if [[ "$(git tag --points-at master)" != "$current" ]] +then + fail "You are not on master" +fi + +version=${current#v} +if ! grep "Version:" ShellCheck.cabal | grep -qFw "$version" +then + fail "The cabal file does not match tag version $version" +fi + +if ! grep -qF "## $current" CHANGELOG.md +then + fail "CHANGELOG.md does not contain '## $current'" +fi + +if [[ $(git log -1 --pretty=%B) != "Stable version "* ]] +then + fail "Expected git log message to be 'Stable version ...'" +fi + +i=1 j=1 +cat << EOF + +Manual Checklist + +$((i++)). Make sure none of the automated checks above failed +$((i++)). Make sure Travis build currently passes: https://travis-ci.org/koalaman/shellcheck +$((i++)). Make sure SnapCraft build currently works: https://build.snapcraft.io/user/koalaman +$((i++)). Run test/distrotest to ensure that most distros can build OOTB. +$((i++)). Format and read over the manual for bad formatting and outdated info. +$((i++)). Make sure the Hackage package builds, so that all files are + +Release Steps + +$((j++)). \`cabal sdist\` to generate a Hackage package +$((j++)). \`git push --follow-tags\` to push commit +$((j++)). Wait for Travis to build +$((j++)). Verify release: + a. Check that the new versions are uploaded: https://shellcheck.storage.googleapis.com/index.html + b. Check that the docker images have version tags: https://hub.docker.com/u/koalaman +$((j++)). If no disaster, upload to Hackage: http://hackage.haskell.org/upload +$((j++)). Push a new commit that updates CHANGELOG.md +EOF +exit "$failed" From b2dd00e4ee7d74774932024924dbe34988d9ab6f Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 28 Jul 2019 17:26:31 -0700 Subject: [PATCH 180/763] Mention aarch64 and macOS binaries in CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 45c9a0e..e65ddb4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ## Since previous release ### Added +- Precompiled binaries for macOS and Linux aarch64 - Preliminary support for fix suggestions - New `-f diff` unified diff format for auto-fixes - Files containing Bats tests can now be checked From 9cc9a575b274d081b59c5384dbab8112380aa9a6 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 28 Jul 2019 18:12:11 -0700 Subject: [PATCH 181/763] Tweak man page --- shellcheck.1.md | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/shellcheck.1.md b/shellcheck.1.md index da66a69..02175dc 100644 --- a/shellcheck.1.md +++ b/shellcheck.1.md @@ -201,6 +201,7 @@ not warn at all, as `ksh` supports decimals in arithmetic contexts. # DIRECTIVES + ShellCheck directives can be specified as comments in the shell script. If they appear before the first command, they are considered file-wide. Otherwise, they apply to the immediately following command or block: @@ -256,6 +257,7 @@ Valid keys are: as a more targeted alternative to 'disable=2039'. # RC FILES + Unless `--norc` is used, ShellCheck will look for a file `.shellcheckrc` or `shellcheckrc` in the script's directory and each parent directory. If found, it will read `key=value` pairs from it and treat them as file-wide directives. @@ -289,6 +291,7 @@ are mounted in the container, so `~/.shellcheckrc` will not be read. # ENVIRONMENT VARIABLES + The environment variable `SHELLCHECK_OPTS` can be set with default flags: export SHELLCHECK_OPTS='--shell=bash --exclude=SC2016' @@ -307,6 +310,7 @@ ShellCheck uses the follow exit codes: + 4: ShellCheck was invoked with bad options (e.g. unknown formatter). # LOCALE + This version of ShellCheck is only available in English. All files are leniently decoded as UTF-8, with a fallback of ISO-8859-1 for invalid sequences. `LC_CTYPE` is respected for output, and defaults to UTF-8 for @@ -315,20 +319,23 @@ locales where encoding is unspecified (such as the `C` locale). Windows users seeing `commitBuffer: invalid argument (invalid character)` should set their terminal to use UTF-8 with `chcp 65001`. -# AUTHOR -ShellCheck is written and maintained by Vidar Holen. +# AUTHORS + +ShellCheck is developed and maintained by Vidar Holen, with assistance from a +long list of wonderful contributors. # REPORTING BUGS + Bugs and issues can be reported on GitHub: https://github.com/koalaman/shellcheck/issues # COPYRIGHT -Copyright 2012-2019, Vidar Holen. + +Copyright 2012-2019, Vidar Holen and contributors. Licensed under the GNU General Public License version 3 or later, see https://gnu.org/licenses/gpl.html - # SEE ALSO sh(1) bash(1) From b7b4d5d29e401858074b0d36d7bb53da58c3932d Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 28 Jul 2019 17:45:01 -0700 Subject: [PATCH 182/763] Stable version 0.7.0 This release is dedicated to RetroArch: the second best way to make your PC feel like a 16bit system (right after building ShellCheck with GHC) --- CHANGELOG.md | 2 +- README.md | 2 ++ ShellCheck.cabal | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e65ddb4..583678e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## Since previous release +## v0.7.0 - 2019-07-28 ### Added - Precompiled binaries for macOS and Linux aarch64 - Preliminary support for fix suggestions diff --git a/README.md b/README.md index 4c9e9ae..9237460 100644 --- a/README.md +++ b/README.md @@ -213,6 +213,8 @@ Alternatively, you can download pre-compiled binaries for the latest release her * [Linux, x86_64](https://storage.googleapis.com/shellcheck/shellcheck-stable.linux.x86_64.tar.xz) (statically linked) * [Linux, armv6hf](https://storage.googleapis.com/shellcheck/shellcheck-stable.linux.armv6hf.tar.xz), i.e. Raspberry Pi (statically linked) +* [Linux, aarch64](https://storage.googleapis.com/shellcheck/shellcheck-stable.linux.armv6hf.tar.xz) aka ARM64 (statically linked) +* [MacOS, x86_64](https://shellcheck.storage.googleapis.com/shellcheck-stable.darwin.x86_64.tar.xz) * [Windows, x86](https://storage.googleapis.com/shellcheck/shellcheck-stable.zip) or see the [storage bucket listing](https://shellcheck.storage.googleapis.com/index.html) for checksums, older versions and the latest daily builds. diff --git a/ShellCheck.cabal b/ShellCheck.cabal index 4658dd0..372bde4 100644 --- a/ShellCheck.cabal +++ b/ShellCheck.cabal @@ -1,5 +1,5 @@ Name: ShellCheck -Version: 0.6.0 +Version: 0.7.0 Synopsis: Shell script analysis tool License: GPL-3 License-file: LICENSE From c175971bf0a144bdc0d57a12e56554933412658e Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 28 Jul 2019 20:50:50 -0700 Subject: [PATCH 183/763] Make `-f diff` stop saying it found more issues when it didn't. --- CHANGELOG.md | 4 ++++ src/ShellCheck/Formatter/Diff.hs | 21 ++++++++++++--------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 583678e..ac5e8f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## v0.7.1 - soon +### Fixed +- `-f diff` no longer claims that it found more issues when it didn't + ## v0.7.0 - 2019-07-28 ### Added - Precompiled binaries for macOS and Linux aarch64 diff --git a/src/ShellCheck/Formatter/Diff.hs b/src/ShellCheck/Formatter/Diff.hs index 445d9de..83fb232 100644 --- a/src/ShellCheck/Formatter/Diff.hs +++ b/src/ShellCheck/Formatter/Diff.hs @@ -43,14 +43,15 @@ ltt x = trace (show x) x format :: FormatterOptions -> IO Formatter format options = do - didOutput <- newIORef False + foundIssues <- newIORef False + reportedIssues <- newIORef False shouldColor <- shouldOutputColor (foColorOption options) let color = if shouldColor then colorize else nocolor return Formatter { header = return (), - footer = checkFooter didOutput color, + footer = checkFooter foundIssues reportedIssues color, onFailure = reportFailure color, - onResult = reportResult didOutput color + onResult = reportResult foundIssues reportedIssues color } @@ -69,9 +70,10 @@ printErr :: ColorFunc -> String -> IO () printErr color = hPutStrLn stderr . color bold . color red reportFailure color file msg = printErr color $ file ++ ": " ++ msg -checkFooter didOutput color = do - output <- readIORef didOutput - unless output $ +checkFooter foundIssues reportedIssues color = do + found <- readIORef foundIssues + output <- readIORef reportedIssues + when (found && not output) $ printErr color "Issues were detected, but none were auto-fixable. Use another format to see them." type ColorFunc = (Int -> String -> String) @@ -79,9 +81,10 @@ data LFStatus = LinefeedMissing | LinefeedOk data DiffDoc a = DiffDoc String LFStatus [DiffRegion a] data DiffRegion a = DiffRegion (Int, Int) (Int, Int) [Diff a] -reportResult :: (IORef Bool) -> ColorFunc -> CheckResult -> SystemInterface IO -> IO () -reportResult didOutput color result sys = do +reportResult :: (IORef Bool) -> (IORef Bool) -> ColorFunc -> CheckResult -> SystemInterface IO -> IO () +reportResult foundIssues reportedIssues color result sys = do let comments = crComments result + unless (null comments) $ writeIORef foundIssues True let suggestedFixes = mapMaybe pcFix comments let fixmap = buildFixMap suggestedFixes mapM_ output $ M.toList fixmap @@ -91,7 +94,7 @@ reportResult didOutput color result sys = do case file of Right contents -> do putStrLn $ formatDoc color $ makeDiff name contents fix - writeIORef didOutput True + writeIORef reportedIssues True Left msg -> reportFailure color name msg hasTrailingLinefeed str = From 3fdc6babb251899f5a5c459e82f74b79d8aec519 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 31 Jul 2019 21:15:46 -0700 Subject: [PATCH 184/763] Update TravisCI config for new winghc docker image --- .compile_binaries | 2 +- .travis.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.compile_binaries b/.compile_binaries index cf6c848..b7878e7 100755 --- a/.compile_binaries +++ b/.compile_binaries @@ -55,7 +55,7 @@ build_armv6hf() { build_windows() { # Windows .exe - docker run --user="$UID" -v "$PWD:/appdata" koalaman/winghc cuib + docker run -v "$PWD:/appdata" koalaman/winghc cuib for tag in $TAGS do cp "dist/build/ShellCheck/shellcheck.exe" "deploy/shellcheck-$tag.exe"; diff --git a/.travis.yml b/.travis.yml index 81fcb2e..6f658a6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,7 +31,7 @@ script: - mkdir -p deploy - source ./.compile_binaries - ./striptests - - set -x; build_"$BUILD"; set +x; + - set -ex; build_"$BUILD"; set +x; - ./.prepare_deploy after_success: | From 71a4053e8cf281cbfd0d6be22ed0ec5e0c1ad051 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 31 Jul 2019 21:32:13 -0700 Subject: [PATCH 185/763] Remove _cleanup now that builds don't run in sequence --- .compile_binaries | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.compile_binaries b/.compile_binaries index b7878e7..d015411 100755 --- a/.compile_binaries +++ b/.compile_binaries @@ -1,9 +1,5 @@ #!/bin/bash -_cleanup(){ - rm -rf dist shellcheck || true -} - build_linux() { # Linux Docker image name="$DOCKER_BASE" @@ -30,7 +26,6 @@ build_linux() { sed -e '/DELETE-MARKER/,$d' Dockerfile > Dockerfile.alpine docker build -f Dockerfile.alpine -t "$name:current" . docker run "$name:current" sh -c 'shellcheck --version' - _cleanup } build_aarch64() { @@ -50,7 +45,6 @@ build_armv6hf() { do cp "shellcheck" "deploy/shellcheck-$tag.linux-armv6hf"; done - _cleanup } build_windows() { @@ -60,7 +54,6 @@ build_windows() { do cp "dist/build/ShellCheck/shellcheck.exe" "deploy/shellcheck-$tag.exe"; done - _cleanup } build_osx() { @@ -77,6 +70,5 @@ build_osx() { do cp "dist/build/shellcheck/shellcheck" "deploy/shellcheck-$tag.darwin-x86_64"; done - _cleanup } From 9423691039f6b3ed91d61dcf94289b46ec35944b Mon Sep 17 00:00:00 2001 From: Glen Mailer Date: Mon, 12 Aug 2019 22:24:47 +0100 Subject: [PATCH 186/763] Mention the CircleCI shellcheck orb in the README. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 9237460..45b5a23 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,7 @@ Services and platforms that have ShellCheck pre-installed and ready to use: * [Codacy](https://www.codacy.com/) * [Code Climate](https://codeclimate.com/) * [Code Factor](https://www.codefactor.io/) +* [CircleCI](https://circleci.com) via the [ShellCheck Orb](https://circleci.com/orbs/registry/orb/circleci/shellcheck) Services and platforms with third party plugins: From e01c4705986b67128158950e54af10792391748b Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 8 Sep 2019 20:06:06 -0700 Subject: [PATCH 187/763] Suggest quoting case patterns, as for SC2053 (fixes #1682) --- CHANGELOG.md | 3 +++ src/ShellCheck/Analytics.hs | 16 ++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac5e8f2..62f88cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ### Fixed - `-f diff` no longer claims that it found more issues when it didn't +### Added +- SC2254: Suggest quoting expansions in case statements + ## v0.7.0 - 2019-07-28 ### Added - Precompiled binaries for macOS and Linux aarch64 diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 067a53f..c07d1a8 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -125,6 +125,7 @@ nodeChecks = [ ,checkArithmeticDeref ,checkArithmeticBadOctal ,checkComparisonAgainstGlob + ,checkCaseAgainstGlob ,checkCommarrays ,checkOrNeq ,checkEchoWc @@ -1338,6 +1339,21 @@ checkComparisonAgainstGlob params (TC_Binary _ SingleBracket op _ word) checkComparisonAgainstGlob _ _ = return () +prop_checkCaseAgainstGlob1 = verify checkCaseAgainstGlob "case foo in lol$n) foo;; esac" +prop_checkCaseAgainstGlob2 = verify checkCaseAgainstGlob "case foo in $(foo)) foo;; esac" +prop_checkCaseAgainstGlob3 = verifyNot checkCaseAgainstGlob "case foo in *$bar*) foo;; esac" +checkCaseAgainstGlob _ t = + case t of + (T_CaseExpression _ _ cases) -> mapM_ check cases + _ -> return () + where + check (_, list, _) = mapM_ check' list + check' expr@(T_NormalWord _ list) + -- If it's already a glob, assume that's what the user wanted + | not (isGlob expr) && any isQuoteableExpansion list = + warn (getId expr) 2254 "Quote expansions in case patterns to match literally rather than as a glob." + check' _ = return () + prop_checkCommarrays1 = verify checkCommarrays "a=(1, 2)" prop_checkCommarrays2 = verify checkCommarrays "a+=(1,2,3)" prop_checkCommarrays3 = verifyNot checkCommarrays "cow=(1 \"foo,bar\" 3)" From ff1eab286c9999e3d85b08e3b466aefcf9ae69a5 Mon Sep 17 00:00:00 2001 From: Renato Assis Date: Wed, 25 Sep 2019 19:20:25 -0300 Subject: [PATCH 188/763] add github --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 9237460..1dc0ec7 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,7 @@ Services and platforms that have ShellCheck pre-installed and ready to use: * [Codacy](https://www.codacy.com/) * [Code Climate](https://codeclimate.com/) * [Code Factor](https://www.codefactor.io/) +* [Github](https://github.com/features/actions)(only Linux) Services and platforms with third party plugins: From de9ab4e6ef5262eeba6871a03ef3938a93b44395 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 28 Sep 2019 14:03:11 -0700 Subject: [PATCH 189/763] Fix glob range duplicate warning in [!!] (fixes #1706) --- src/ShellCheck/Analytics.hs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index c07d1a8..5e0adf7 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2341,6 +2341,7 @@ prop_checkCharRangeGlob2 = verifyNot checkCharRangeGlob "ls *[[:digit:]].jpg" prop_checkCharRangeGlob3 = verify checkCharRangeGlob "ls [10-15]" prop_checkCharRangeGlob4 = verifyNot checkCharRangeGlob "ls [a-zA-Z]" prop_checkCharRangeGlob5 = verifyNot checkCharRangeGlob "tr -d [a-zA-Z]" -- tr has 2060 +prop_checkCharRangeGlob6 = verifyNot checkCharRangeGlob "[[ $x == [!!]* ]]" checkCharRangeGlob p t@(T_Glob id str) | isCharClass str && not (isParamTo (parentMap p) "tr" t) = if ":" `isPrefixOf` contents @@ -2352,8 +2353,13 @@ checkCharRangeGlob p t@(T_Glob id str) | info id 2102 "Ranges can only match single chars (mentioned due to duplicates)." where isCharClass str = "[" `isPrefixOf` str && "]" `isSuffixOf` str - contents = drop 1 . take (length str - 1) $ str + contents = dropNegation . drop 1 . take (length str - 1) $ str hasDupes = any (>1) . map length . group . sort . filter (/= '-') $ contents + dropNegation s = + case s of + '!':rest -> rest + '^':rest -> rest + x -> x checkCharRangeGlob _ _ = return () From 7fb399528cb9189b29dc92110fab76fcf5d81886 Mon Sep 17 00:00:00 2001 From: Supanat Pothivarakorn Date: Wed, 2 Oct 2019 22:34:43 +0700 Subject: [PATCH 190/763] Allow `read -t 0` to not require -r flag since it has specific purpose for checking only --- src/ShellCheck/Analytics.hs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 5e0adf7..9876679 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2786,8 +2786,11 @@ checkMaskedReturns _ _ = return () prop_checkReadWithoutR1 = verify checkReadWithoutR "read -a foo" prop_checkReadWithoutR2 = verifyNot checkReadWithoutR "read -ar foo" +prop_checkReadWithoutR3 = verifyNot checkReadWithoutR "read -t 0" +prop_checkReadWithoutR4 = verifyNot checkReadWithoutR "read -t 0 && read --d '' -r bar" +prop_checkReadWithoutR5 = verify checkReadWithoutR "read -t 0 foo < file.txt" checkReadWithoutR _ t@T_SimpleCommand {} | t `isUnqualifiedCommand` "read" = - unless ("r" `elem` map snd (getAllFlags t)) $ + unless (oversimplify t == ["read", "-t", "0"] || "r" `elem` map snd (getAllFlags t)) $ info (getId $ getCommandTokenOrThis t) 2162 "read without -r will mangle backslashes." checkReadWithoutR _ _ = return () From fa0f88c106afb4defe35ea9b17f3bc4f884a7d04 Mon Sep 17 00:00:00 2001 From: ryantig Date: Fri, 4 Oct 2019 11:11:21 -0700 Subject: [PATCH 191/763] Update README.md Repair link to #installing-a-pre-compiled-binary (was pointing to #installing-the-shellcheck-binary) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9237460..8ad5d10 100644 --- a/README.md +++ b/README.md @@ -116,7 +116,7 @@ Services and platforms with third party plugins: Most other services, including [GitLab](https://about.gitlab.com/), let you install ShellCheck yourself, either through the system's package manager (see [Installing](#installing)), -or by downloading and unpacking a [binary release](#installing-the-shellcheck-binary). +or by downloading and unpacking a [binary release](#installing-a-pre-compiled-binary). It's a good idea to manually install a specific ShellCheck version regardless. This avoids any surprise build breaks when a new version with new warnings is published. From afea62de4e115d7badd37bd5d0ec7af63764240f Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 12 Oct 2019 19:55:20 -0700 Subject: [PATCH 192/763] Suggest using `$((..))` in `[ 2*3 -eq 6 ]` (fixes #1641) --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 25 +++++++++++++++++++++++-- src/ShellCheck/Data.hs | 4 ++++ 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 62f88cf..48f960f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Added - SC2254: Suggest quoting expansions in case statements +- SC2255: Suggest using `$((..))` in `[ 2*3 -eq 6 ]` ## v0.7.0 - 2019-07-28 ### Added diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 5e0adf7..db1c338 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1044,7 +1044,7 @@ checkNumberComparisons params (TC_Binary id typ op lhs rhs) = do Dash -> err id 2073 $ "Escape \\" ++ op ++ " to prevent it redirecting." _ -> err id 2073 $ "Escape \\" ++ op ++ " to prevent it redirecting (or switch to [[ .. ]])." - when (op `elem` ["-lt", "-gt", "-le", "-ge", "-eq"]) $ do + when (op `elem` arithmeticBinaryTestOps) $ do mapM_ checkDecimals [lhs, rhs] when (typ == SingleBracket) $ checkStrings [lhs, rhs] @@ -1193,7 +1193,7 @@ checkConstantIfs _ (TC_Binary id typ op lhs rhs) | not isDynamic = else checkUnmatchable id op lhs rhs where isDynamic = - op `elem` [ "-lt", "-gt", "-le", "-ge", "-eq", "-ne" ] + op `elem` arithmeticBinaryTestOps && typ == DoubleBracket || op `elem` [ "-nt", "-ot", "-ef"] @@ -2700,6 +2700,10 @@ prop_checkTestArgumentSplitting15 = verifyNot checkTestArgumentSplitting "[[ \"$ prop_checkTestArgumentSplitting16 = verifyNot checkTestArgumentSplitting "[[ -v foo[123] ]]" prop_checkTestArgumentSplitting17 = verifyNot checkTestArgumentSplitting "#!/bin/ksh\n[ -e foo* ]" prop_checkTestArgumentSplitting18 = verify checkTestArgumentSplitting "#!/bin/ksh\n[ -d foo* ]" +prop_checkTestArgumentSplitting19 = verifyNot checkTestArgumentSplitting "[[ var[x] -eq 2*3 ]]" +prop_checkTestArgumentSplitting20 = verify checkTestArgumentSplitting "[ var[x] -eq 2 ]" +prop_checkTestArgumentSplitting21 = verify checkTestArgumentSplitting "[ 6 -eq 2*3 ]" +prop_checkTestArgumentSplitting22 = verify checkTestArgumentSplitting "[ foo -eq 'y' ]" checkTestArgumentSplitting :: Parameters -> Token -> Writer [TokenComment] () checkTestArgumentSplitting params t = case t of @@ -2729,6 +2733,18 @@ checkTestArgumentSplitting params t = (TC_Unary _ typ op token) -> checkAll typ token + (TC_Binary _ typ op lhs rhs) | op `elem` arithmeticBinaryTestOps -> + if typ == DoubleBracket + then + mapM_ (\c -> do + checkArrays typ c + checkBraces typ c) [lhs, rhs] + else + mapM_ (\c -> do + checkNumericalGlob typ c + checkArrays typ c + checkBraces typ c) [lhs, rhs] + (TC_Binary _ typ op lhs rhs) -> if op `elem` ["=", "==", "!=", "=~"] then do @@ -2761,6 +2777,11 @@ checkTestArgumentSplitting params t = then warn (getId token) 2202 "Globs don't work as operands in [ ]. Use a loop." else err (getId token) 2203 "Globs are ignored in [[ ]] except right of =/!=. Use a loop." + checkNumericalGlob SingleBracket token = + -- var[x] and x*2 look like globs + when (shellType params /= Ksh && isGlob token) $ + err (getId token) 2255 "[ ] does not apply arithmetic evaluation. Evaluate with $((..)) for numbers, or use string comparator for strings." + prop_checkMaskedReturns1 = verify checkMaskedReturns "f() { local a=$(false); }" prop_checkMaskedReturns2 = verify checkMaskedReturns "declare a=$(false)" diff --git a/src/ShellCheck/Data.hs b/src/ShellCheck/Data.hs index 1394c04..e2eeb74 100644 --- a/src/ShellCheck/Data.hs +++ b/src/ShellCheck/Data.hs @@ -114,6 +114,10 @@ binaryTestOps = [ "-gt", "-ge", "=~", ">", "<", "=", "\\<", "\\>", "\\<=", "\\>=" ] +arithmeticBinaryTestOps = [ + "-eq", "-ne", "-lt", "-le", "-gt", "-ge" + ] + unaryTestOps = [ "!", "-a", "-b", "-c", "-d", "-e", "-f", "-g", "-h", "-L", "-k", "-p", "-r", "-s", "-S", "-t", "-u", "-w", "-x", "-O", "-G", "-N", "-z", "-n", From 7473d4a7434fb7c0ff3b0c9cbc2455f2513d0ce7 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 12 Oct 2019 20:45:36 -0700 Subject: [PATCH 193/763] Make `read -t 0` test more forgiving towards other flags --- src/ShellCheck/Analytics.hs | 13 +++++++++++-- src/ShellCheck/AnalyzerLib.hs | 11 ++++++----- src/ShellCheck/Checks/Commands.hs | 2 +- src/ShellCheck/Data.hs | 2 ++ 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index f778d34..a182afc 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2809,10 +2809,19 @@ prop_checkReadWithoutR1 = verify checkReadWithoutR "read -a foo" prop_checkReadWithoutR2 = verifyNot checkReadWithoutR "read -ar foo" prop_checkReadWithoutR3 = verifyNot checkReadWithoutR "read -t 0" prop_checkReadWithoutR4 = verifyNot checkReadWithoutR "read -t 0 && read --d '' -r bar" -prop_checkReadWithoutR5 = verify checkReadWithoutR "read -t 0 foo < file.txt" +prop_checkReadWithoutR5 = verifyNot checkReadWithoutR "read -t 0 foo < file.txt" +prop_checkReadWithoutR6 = verifyNot checkReadWithoutR "read -u 3 -t 0" checkReadWithoutR _ t@T_SimpleCommand {} | t `isUnqualifiedCommand` "read" = - unless (oversimplify t == ["read", "-t", "0"] || "r" `elem` map snd (getAllFlags t)) $ + unless ("r" `elem` map snd flags || has_t0) $ info (getId $ getCommandTokenOrThis t) 2162 "read without -r will mangle backslashes." + where + flags = getAllFlags t + has_t0 = fromMaybe False $ do + parsed <- getOpts flagsForRead flags + t <- getOpt "t" parsed + str <- getLiteralString t + return $ str == "0" + checkReadWithoutR _ _ = return () prop_checkUncheckedCd1 = verifyTree checkUncheckedCdPushdPopd "cd ~/src; rm -r foo" diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 70b781e..7803fdf 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -932,12 +932,11 @@ isQuotedAlternativeReference t = -- Just [("r", -re), ("e", -re), ("d", :), ("u", 3), ("", bar)] -- where flags with arguments map to arguments, while others map to themselves. -- Any unrecognized flag will result in Nothing. -getGnuOpts = getOpts getAllFlags -getBsdOpts = getOpts getLeadingFlags -getOpts :: (Token -> [(Token, String)]) -> String -> Token -> Maybe [(String, Token)] -getOpts flagTokenizer string cmd = process flags +getGnuOpts str t = getOpts str $ getAllFlags t +getBsdOpts str t = getOpts str $ getLeadingFlags t +getOpts :: String -> [(Token, String)] -> Maybe [(String, Token)] +getOpts string flags = process flags where - flags = flagTokenizer cmd flagList (c:':':rest) = ([c], True) : flagList rest flagList (c:rest) = ([c], False) : flagList rest flagList [] = [] @@ -959,6 +958,8 @@ getOpts flagTokenizer string cmd = process flags more <- process rest2 return $ (flag1, token1) : more +getOpt str flags = snd <$> (listToMaybe $ filter (\(f, _) -> f == str) $ flags) + supportsArrays shell = shell == Bash || shell == Ksh -- Returns true if the shell is Bash or Ksh (sorry for the name, Ksh) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index c6346a9..441e43f 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -676,7 +676,7 @@ prop_checkReadExpansions7 = verifyNot checkReadExpansions "read $1" prop_checkReadExpansions8 = verifyNot checkReadExpansions "read ${var?}" checkReadExpansions = CommandCheck (Exactly "read") check where - options = getGnuOpts "sreu:n:N:i:p:a:" + options = getGnuOpts flagsForRead getVars cmd = fromMaybe [] $ do opts <- options cmd return . map snd $ filter (\(x,_) -> x == "" || x == "a") opts diff --git a/src/ShellCheck/Data.hs b/src/ShellCheck/Data.hs index e2eeb74..732619d 100644 --- a/src/ShellCheck/Data.hs +++ b/src/ShellCheck/Data.hs @@ -136,3 +136,5 @@ shellForExecutable name = "ksh88" -> return Ksh "ksh93" -> return Ksh otherwise -> Nothing + +flagsForRead = "sreu:n:N:i:p:a:t:" From 764fdcb260d6f77892fce7306357385d637437cd Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 12 Oct 2019 20:48:22 -0700 Subject: [PATCH 194/763] Move failing test to correct check --- src/ShellCheck/Analytics.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index db1c338..0c1503d 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1021,6 +1021,7 @@ prop_checkNumberComparisons12 = verify checkNumberComparisons "[ x$foo -gt x${N} prop_checkNumberComparisons13 = verify checkNumberComparisons "[ $foo > $bar ]" prop_checkNumberComparisons14 = verifyNot checkNumberComparisons "[[ foo < bar ]]" prop_checkNumberComparisons15 = verifyNot checkNumberComparisons "[ $foo '>' $bar ]" +prop_checkNumberComparisons16 = verify checkNumberComparisons "[ foo -eq 'y' ]" checkNumberComparisons params (TC_Binary id typ op lhs rhs) = do if isNum lhs || isNum rhs then do @@ -2703,7 +2704,6 @@ prop_checkTestArgumentSplitting18 = verify checkTestArgumentSplitting "#!/bin/ks prop_checkTestArgumentSplitting19 = verifyNot checkTestArgumentSplitting "[[ var[x] -eq 2*3 ]]" prop_checkTestArgumentSplitting20 = verify checkTestArgumentSplitting "[ var[x] -eq 2 ]" prop_checkTestArgumentSplitting21 = verify checkTestArgumentSplitting "[ 6 -eq 2*3 ]" -prop_checkTestArgumentSplitting22 = verify checkTestArgumentSplitting "[ foo -eq 'y' ]" checkTestArgumentSplitting :: Parameters -> Token -> Writer [TokenComment] () checkTestArgumentSplitting params t = case t of From 60f75e5b8a5460812c6bb89099ad562229ca0e17 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 13 Oct 2019 20:26:40 -0700 Subject: [PATCH 195/763] Warn about unexpected characters after ]/]] (fixes #1680) --- src/ShellCheck/Parser.hs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index b2935fd..075d486 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -910,6 +910,7 @@ prop_readCondition20 = isOk readCondition "[[ echo_rc -eq 0 ]]" prop_readCondition21 = isOk readCondition "[[ $1 =~ ^(a\\ b)$ ]]" prop_readCondition22 = isOk readCondition "[[ $1 =~ \\.a\\.(\\.b\\.)\\.c\\. ]]" prop_readCondition23 = isOk readCondition "[[ -v arr[$var] ]]" +prop_readCondition24 = isWarning readCondition "[[ 1 == 2 ]]]" readCondition = called "test expression" $ do opos <- getPosition start <- startSpan @@ -938,6 +939,11 @@ readCondition = called "test expression" $ do id <- endSpan start when (open == "[[" && close /= "]]") $ parseProblemAt cpos ErrorC 1033 "Did you mean ]] ?" when (open == "[" && close /= "]" ) $ parseProblemAt opos ErrorC 1034 "Did you mean [[ ?" + optional $ lookAhead $ do + pos <- getPosition + notFollowedBy2 readCmdWord <|> + parseProblemAt pos ErrorC 1136 + ("Unexpected characters after terminating " ++ close ++ ". Missing semicolon/linefeed?") spacing many readCmdWord -- Read and throw away remainders to get then/do warnings. Fixme? return $ T_Condition id typ condition From 79ba67dbd38d05bbfe0287507218a57020f2586a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Merlin=20G=C3=B6ttlinger?= Date: Mon, 21 Oct 2019 08:04:59 +0200 Subject: [PATCH 196/763] Nix install instructions --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 9237460..314e0d4 100644 --- a/README.md +++ b/README.md @@ -209,6 +209,11 @@ docker run --rm -v "$PWD:/mnt" koalaman/shellcheck:stable myscript or use `koalaman/shellcheck-alpine` if you want a larger Alpine Linux based image to extend. It works exactly like a regular Alpine image, but has shellcheck preinstalled. +Using the [nix package manager](https://nixos.org/nix): +```sh +nix-env -iA nixpkgs.shellcheck +``` + Alternatively, you can download pre-compiled binaries for the latest release here: * [Linux, x86_64](https://storage.googleapis.com/shellcheck/shellcheck-stable.linux.x86_64.tar.xz) (statically linked) From 4dfd7eb1cfafb920f23c1fd509de7e075246fcb8 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 24 Oct 2019 10:33:17 -0700 Subject: [PATCH 197/763] Use single quotes for the format string example in SC2059 --- src/ShellCheck/Checks/Commands.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 441e43f..7f06862 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -573,7 +573,7 @@ checkPrintfVar = CommandCheck (Exactly "printf") (f . arguments) where unless ('%' `elem` concat (oversimplify format) || isLiteral format) $ info (getId format) 2059 - "Don't use variables in the printf format string. Use printf \"..%s..\" \"$foo\"." + "Don't use variables in the printf format string. Use printf '..%s..' \"$foo\"." where onlyTrailingTs format argCount = all (== 'T') $ drop argCount format From 30c75340e6ca5dddbae8977caebe671bf77f12d3 Mon Sep 17 00:00:00 2001 From: "gabriele.lana" Date: Sat, 26 Oct 2019 15:41:46 +0200 Subject: [PATCH 198/763] Parse regular `for` with body in curly braces Fixes #1694 --- src/ShellCheck/Parser.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 075d486..b2717aa 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -2448,6 +2448,7 @@ readDoGroup kwId = do prop_readForClause = isOk readForClause "for f in *; do rm \"$f\"; done" +prop_readForClause1 = isOk readForClause "for f in *; { rm \"$f\"; }" prop_readForClause3 = isOk readForClause "for f; do foo; done" prop_readForClause4 = isOk readForClause "for((i=0; i<10; i++)); do echo $i; done" prop_readForClause5 = isOk readForClause "for ((i=0;i<10 && n>x;i++,--n))\ndo \necho $i\ndone" @@ -2487,7 +2488,7 @@ readForClause = called "for loop" $ do "Don't use $ on the iterator name in for loops." name <- readVariableName `thenSkip` allspacing values <- readInClause <|> (optional readSequentialSep >> return []) - group <- readDoGroup id + group <- readBraced <|> readDoGroup id return $ T_ForIn id name values group prop_readSelectClause1 = isOk readSelectClause "select foo in *; do echo $foo; done" @@ -3431,4 +3432,3 @@ tryWithErrors parser = do return [] runTests = $quickCheckAll - From 699aac589acb38445a76c95ba1602211cb6204a5 Mon Sep 17 00:00:00 2001 From: "gabriele.lana" Date: Sat, 26 Oct 2019 17:36:32 +0200 Subject: [PATCH 199/763] Support for heredoc quoted token like `'"FOO"` Fixes #1650 --- src/ShellCheck/Parser.hs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 075d486..d4dabe6 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -1739,6 +1739,7 @@ prop_readHereDoc14= isWarning readScript "cat << foo\nbar\nfoo \n" prop_readHereDoc15= isWarning readScript "cat < (Quoted, String) + unquote "" = (Unquoted, "") + unquote [c] = (Unquoted, [c]) + unquote s@(cl:tl) = + case reverse tl of + (cr:tr) | cr == cl && cl `elem` "\"'" -> (Quoted, reverse tr) + _ -> (if '\\' `elem` s then (Quoted, filter ((/=) '\\') s) else (Unquoted, s)) -- Fun fact: bash considers << foo"" quoted, but not << <("foo"). - -- Instead of replicating this, just read a token and strip quotes. readToken = do str <- readStringForParser readNormalWord - return (if any (`elem` quotes) str then Quoted else Unquoted, - filter (not . (`elem` quotes)) str) - + return $ unquote str readPendingHereDocs = do docs <- popPendingHereDocs From 0e0de940450425891d1f5034f761bede1afe6559 Mon Sep 17 00:00:00 2001 From: Tito Sacchi Date: Thu, 31 Oct 2019 17:34:10 +0100 Subject: [PATCH 200/763] Fix issue #1724 (bash: missing support for 'builtin' keyword) Now shellcheck looks for the arguments to 'builtin' to determine read/written variables. A change in the parser makes sure that assignments are parsed correctly in commands that start with 'builtin'. --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 1 + src/ShellCheck/AnalyzerLib.hs | 4 +++- src/ShellCheck/Parser.hs | 10 +++++++++- 4 files changed, 14 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 48f960f..79ca5f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## v0.7.1 - soon ### Fixed - `-f diff` no longer claims that it found more issues when it didn't +- SC2154 triggers for builtins called with `builtin` ### Added - SC2254: Suggest quoting expansions in case statements diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 221caa1..4448796 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2168,6 +2168,7 @@ prop_checkUnassignedReferences35= verifyNotTree checkUnassignedReferences "echo prop_checkUnassignedReferences36= verifyNotTree checkUnassignedReferences "read -a foo -r <<<\"foo bar\"; echo \"$foo\"" prop_checkUnassignedReferences37= verifyNotTree checkUnassignedReferences "var=howdy; printf -v 'array[0]' %s \"$var\"; printf %s \"${array[0]}\";" prop_checkUnassignedReferences38= verifyTree (checkUnassignedReferences' True) "echo $VAR" +prop_checkUnassignedReferences39= verifyNotTree checkUnassignedReferences "builtin export var=4; echo $var" checkUnassignedReferences = checkUnassignedReferences' False checkUnassignedReferences' includeGlobals params t = warnings diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 7803fdf..590889c 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -567,9 +567,11 @@ getReferencedVariableCommand _ = [] -- VariableName :: String, -- The variable name, i.e. foo -- VariableValue :: DataType -- A description of the value being assigned, i.e. "Literal string with value foo" -- ) -getModifiedVariableCommand base@(T_SimpleCommand _ _ (T_NormalWord _ (T_Literal _ x:_):rest)) = +getModifiedVariableCommand base@(T_SimpleCommand id cmdPrefix (T_NormalWord _ (T_Literal _ x:_):rest)) = filter (\(_,_,s,_) -> not ("-" `isPrefixOf` s)) $ case x of + "builtin" -> + getModifiedVariableCommand $ T_SimpleCommand id cmdPrefix rest "read" -> let params = map getLiteral rest readArrayVars = getReadArrayVariables rest diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 075d486..e42762b 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -246,6 +246,10 @@ addParseNote n = do parseNotes = n : parseNotes state } +ignoreProblemsOf p = do + systemState <- lift . lift $ Ms.get + p <* (lift . lift . Ms.put $ systemState) + shouldIgnoreCode code = do context <- getCurrentContexts checkSourced <- Mr.asks checkSourced @@ -2041,7 +2045,11 @@ readSimpleCommand = called "simple command" $ do Just cmd -> do validateCommand cmd - suffix <- option [] $ getParser readCmdSuffix cmd [ + -- We have to ignore possible parsing problems from the lookAhead parser + firstArgument <- ignoreProblemsOf . optionMaybe . try . lookAhead $ readCmdWord + suffix <- option [] $ getParser readCmdSuffix + -- If `export` or other modifier commands are called with `builtin` we have to look at the first argument + (if isCommand ["builtin"] cmd && isJust firstArgument then fromJust firstArgument else cmd) [ (["declare", "export", "local", "readonly", "typeset"], readModifierSuffix), (["time"], readTimeSuffix), (["let"], readLetSuffix), From 84ca7711c45da3083b18a5643460b620380ac143 Mon Sep 17 00:00:00 2001 From: Tito Sacchi Date: Fri, 1 Nov 2019 14:28:00 +0100 Subject: [PATCH 201/763] Make command-specific checks act on `builtin ...` Now if shellchecks encounters a command like `builtin cmd ...` it applies the same check that would be applied to `cmd ...`. --- src/ShellCheck/Checks/Commands.hs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 441e43f..4841ddd 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -105,12 +105,16 @@ buildCommandMap = foldl' addCheck Map.empty checkCommand :: Map.Map CommandName (Token -> Analysis) -> Token -> Analysis -checkCommand map t@(T_SimpleCommand id _ (cmd:rest)) = fromMaybe (return ()) $ do +checkCommand map t@(T_SimpleCommand id cmdPrefix (cmd:rest)) = fromMaybe (return ()) $ do name <- getLiteralString cmd return $ if '/' `elem` name then Map.findWithDefault nullCheck (Basename $ basename name) map t + else if name == "builtin" && not (null rest) then + let t' = T_SimpleCommand id cmdPrefix rest + selectedBuiltin = fromMaybe "" $ getLiteralString . head $ rest + in Map.findWithDefault nullCheck (Exactly selectedBuiltin) map t' else do Map.findWithDefault nullCheck (Exactly name) map t Map.findWithDefault nullCheck (Basename name) map t From 5becc673b2477b98073e583fe505c4d68e4b945f Mon Sep 17 00:00:00 2001 From: Tito Sacchi Date: Fri, 1 Nov 2019 14:36:15 +0100 Subject: [PATCH 202/763] Modify CHANGELOG.md --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 79ca5f3..8b9aae5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,8 @@ ## v0.7.1 - soon ### Fixed - `-f diff` no longer claims that it found more issues when it didn't -- SC2154 triggers for builtins called with `builtin` +- SC2154 and all command-specific checks now trigger for builtins + called with `builtin` ### Added - SC2254: Suggest quoting expansions in case statements From 5962b01816b4cab4ad612ea4204d13903cace0b0 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 3 Nov 2019 12:45:13 -0800 Subject: [PATCH 203/763] Correctly handle empty variables for SC2086 (fixes #1722) --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 60 ++++++++++++++++++++++++------------- 2 files changed, 40 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 48f960f..6689dac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## v0.7.1 - soon ### Fixed - `-f diff` no longer claims that it found more issues when it didn't +- Known empty variables now correctly trigger SC2086 ### Added - SC2254: Suggest quoting expansions in case statements diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 221caa1..97054c1 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1807,6 +1807,21 @@ prop_checkSpacefulness35= verifyNotTree checkSpacefulness "echo ${1+\"$1\"}" prop_checkSpacefulness36= verifyNotTree checkSpacefulness "arg=$#; echo $arg" prop_checkSpacefulness37= verifyNotTree checkSpacefulness "@test 'status' {\n [ $status -eq 0 ]\n}" prop_checkSpacefulness37v = verifyTree checkVerboseSpacefulness "@test 'status' {\n [ $status -eq 0 ]\n}" +prop_checkSpacefulness38= verifyTree checkSpacefulness "a=; echo $a" +prop_checkSpacefulness39= verifyNotTree checkSpacefulness "a=''\"\"''; b=x$a; echo $b" +prop_checkSpacefulness40= verifyNotTree checkSpacefulness "a=$((x+1)); echo $a" + +data SpaceStatus = SpaceSome | SpaceNone | SpaceEmpty deriving (Eq) +instance Semigroup SpaceStatus where + (<>) x y = + case (x,y) of + (SpaceNone, SpaceNone) -> SpaceNone + (SpaceSome, _) -> SpaceSome + (_, SpaceSome) -> SpaceSome + (SpaceEmpty, x) -> x + (x, SpaceEmpty) -> x +instance Monoid SpaceStatus where + mempty = SpaceEmpty -- This is slightly awkward because we want to support structured -- optional checks based on nearly the same logic @@ -1814,7 +1829,7 @@ checkSpacefulness params = checkSpacefulness' onFind params where emit x = tell [x] onFind spaces token _ = - when spaces $ + when (spaces /= SpaceNone) $ if isDefaultAssignment (parentMap params) token then emit $ makeComment InfoC (getId token) 2223 @@ -1829,7 +1844,6 @@ checkSpacefulness params = checkSpacefulness' onFind params any (`isPrefixOf` modifier) ["=", ":="] && isParamTo parents ":" token - prop_checkSpacefulness4v= verifyTree checkVerboseSpacefulness "foo=3; foo=$(echo $foo)" prop_checkSpacefulness8v= verifyTree checkVerboseSpacefulness "a=foo\\ bar; a=foo; rm $a" prop_checkSpacefulness28v = verifyTree checkVerboseSpacefulness "exec {n}>&1; echo $n" @@ -1837,24 +1851,24 @@ prop_checkSpacefulness36v = verifyTree checkVerboseSpacefulness "arg=$#; echo $a checkVerboseSpacefulness params = checkSpacefulness' onFind params where onFind spaces token name = - when (not spaces && name `notElem` specialVariablesWithoutSpaces) $ + when (spaces == SpaceNone && name `notElem` specialVariablesWithoutSpaces) $ tell [makeCommentWithFix StyleC (getId token) 2248 "Prefer double quoting even when variables don't contain special characters." (addDoubleQuotesAround params token)] addDoubleQuotesAround params token = (surroundWidth (getId token) params "\"") checkSpacefulness' - :: (Bool -> Token -> String -> Writer [TokenComment] ()) -> + :: (SpaceStatus -> Token -> String -> Writer [TokenComment] ()) -> Parameters -> Token -> [TokenComment] checkSpacefulness' onFind params t = doVariableFlowAnalysis readF writeF (Map.fromList defaults) (variableFlow params) where - defaults = zip variablesWithoutSpaces (repeat False) + defaults = zip variablesWithoutSpaces (repeat SpaceNone) - hasSpaces name = gets (Map.findWithDefault True name) + hasSpaces name = gets (Map.findWithDefault SpaceSome name) - setSpaces name bool = - modify $ Map.insert name bool + setSpaces name status = + modify $ Map.insert name status readF _ token name = do spaces <- hasSpaces name @@ -1871,13 +1885,13 @@ checkSpacefulness' onFind params t = where emit x = tell [x] - writeF _ _ name (DataString SourceExternal) = setSpaces name True >> return [] - writeF _ _ name (DataString SourceInteger) = setSpaces name False >> return [] + writeF _ _ name (DataString SourceExternal) = setSpaces name SpaceSome >> return [] + writeF _ _ name (DataString SourceInteger) = setSpaces name SpaceNone >> return [] writeF _ _ name (DataString (SourceFrom vals)) = do map <- get setSpaces name - (isSpacefulWord (\x -> Map.findWithDefault True x map) vals) + (isSpacefulWord (\x -> Map.findWithDefault SpaceSome x map) vals) return [] writeF _ _ _ _ = return [] @@ -1889,24 +1903,28 @@ checkSpacefulness' onFind params t = (T_DollarBraced _ _ _ ) -> True _ -> False - isSpacefulWord :: (String -> Bool) -> [Token] -> Bool - isSpacefulWord f = any (isSpaceful f) - isSpaceful :: (String -> Bool) -> Token -> Bool + isSpacefulWord :: (String -> SpaceStatus) -> [Token] -> SpaceStatus + isSpacefulWord f = mconcat . map (isSpaceful f) + isSpaceful :: (String -> SpaceStatus) -> Token -> SpaceStatus isSpaceful spacefulF x = case x of - T_DollarExpansion _ _ -> True - T_Backticked _ _ -> True - T_Glob _ _ -> True - T_Extglob {} -> True - T_Literal _ s -> s `containsAny` globspace - T_SingleQuoted _ s -> s `containsAny` globspace + T_DollarExpansion _ _ -> SpaceSome + T_Backticked _ _ -> SpaceSome + T_Glob _ _ -> SpaceSome + T_Extglob {} -> SpaceSome + T_DollarArithmetic _ _ -> SpaceNone + T_Literal _ s -> fromLiteral s + T_SingleQuoted _ s -> fromLiteral s T_DollarBraced _ _ _ -> spacefulF $ getBracedReference $ bracedString x T_NormalWord _ w -> isSpacefulWord spacefulF w T_DoubleQuoted _ w -> isSpacefulWord spacefulF w - _ -> False + _ -> SpaceEmpty where globspace = "*?[] \t\n" containsAny s = any (`elem` s) + fromLiteral "" = SpaceEmpty + fromLiteral s | s `containsAny` globspace = SpaceSome + fromLiteral _ = SpaceNone prop_CheckVariableBraces1 = verify checkVariableBraces "a='123'; echo $a" prop_CheckVariableBraces2 = verifyNot checkVariableBraces "a='123'; echo ${a}" From e701cf6fadd3b7c01231f6faac0bd18b27d72448 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 3 Nov 2019 13:25:35 -0800 Subject: [PATCH 204/763] Warn about [ x -ot y ] in POSIX mode --- src/ShellCheck/Checks/ShellSupport.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 83d23fb..07dfdda 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -205,7 +205,7 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do | op `elem` [ "<", ">", "\\<", "\\>", "<=", ">=", "\\<=", "\\>="] = unless isDash $ warnMsg id $ "lexicographical " ++ op ++ " is" bashism (TC_Binary id SingleBracket op _ _) - | op `elem` [ "-nt", "-ef" ] = + | op `elem` [ "-ot", "-nt", "-ef" ] = unless isDash $ warnMsg id $ op ++ " is" bashism (TC_Binary id SingleBracket "==" _ _) = warnMsg id "== in place of = is" From 93eca1cb8e7e8c1428a1375e985a669b43c41e95 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 3 Nov 2019 13:25:46 -0800 Subject: [PATCH 205/763] Only trigger SC1014 when command is a complete word (fixes #1737) --- src/ShellCheck/Parser.hs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 075d486..96822a3 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -437,6 +437,7 @@ readConditionContents single = readCondContents `attempting` lookAhead (do pos <- getPosition s <- readVariableName + spacing1 when (s `elem` commonCommands) $ parseProblemAt pos WarningC 1014 "Use 'if cmd; then ..' to check exit code, or 'if [[ $(cmd) == .. ]]' to check output.") @@ -911,6 +912,7 @@ prop_readCondition21 = isOk readCondition "[[ $1 =~ ^(a\\ b)$ ]]" prop_readCondition22 = isOk readCondition "[[ $1 =~ \\.a\\.(\\.b\\.)\\.c\\. ]]" prop_readCondition23 = isOk readCondition "[[ -v arr[$var] ]]" prop_readCondition24 = isWarning readCondition "[[ 1 == 2 ]]]" +prop_readCondition25 = isOk readCondition "[[ lex.yy.c -ot program.l ]]" readCondition = called "test expression" $ do opos <- getPosition start <- startSpan From 7eb6b35cb04bcf460f21a96db214fcae1fd0c51a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Laurent=20VERDO=C3=8FA?= Date: Sat, 9 Nov 2019 10:26:59 +0100 Subject: [PATCH 206/763] Make image build process a bit simpler Take full leverage of multi-stage docker build. --- .compile_binaries | 3 +-- Dockerfile | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.compile_binaries b/.compile_binaries index d015411..2585c14 100755 --- a/.compile_binaries +++ b/.compile_binaries @@ -23,8 +23,7 @@ build_linux() { # Linux Alpine based Docker image name="$DOCKER_BASE-alpine" DOCKER_BUILDS="$DOCKER_BUILDS $name" - sed -e '/DELETE-MARKER/,$d' Dockerfile > Dockerfile.alpine - docker build -f Dockerfile.alpine -t "$name:current" . + docker build -f Dockerfile -t "$name:current" --target alpine . docker run "$name:current" sh -c 'shellcheck --version' } diff --git a/Dockerfile b/Dockerfile index 2b65291..6ca370e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,7 +22,7 @@ RUN mkdir -p /out/bin && \ cp shellcheck /out/bin/ # Resulting Alpine image -FROM alpine:latest +FROM alpine:latest AS alpine LABEL maintainer="Vidar Holen " COPY --from=build /out / From 2341a4c6836c0eec0a622ed9d3cf83f36ab4c1ab Mon Sep 17 00:00:00 2001 From: Benjamin Gordon Date: Wed, 13 Nov 2019 15:50:21 -0700 Subject: [PATCH 207/763] SC2256: Check for translated strings matching known variables SC2247 already warns about translated strings that look like $"(foo)" or $"{foo}". Since typical use of translated strings is to translate whole messages, a string like $"foo" is likely to be a similar mistake if foo is the name of an existing variable. Conversely, a string like $"foo bar" is potentially meant to be a message id even if foo is a known variable. Add a warning for the $"foo" case, but make it separate from the existing warning so that projects that reuse variable names as their message ids can separately disable the new warning. --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6689dac..d7e4181 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ ### Added - SC2254: Suggest quoting expansions in case statements - SC2255: Suggest using `$((..))` in `[ 2*3 -eq 6 ]` +- SC2256: Warn about translated strings that are known variables ## v0.7.0 - 2019-07-28 ### Added diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 97054c1..b0928ce 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -192,6 +192,7 @@ nodeChecks = [ ,checkRedirectionToCommand ,checkDollarQuoteParen ,checkUselessBang + ,checkTranslatedString ] optionalChecks = map fst optionalTreeChecks @@ -3451,6 +3452,24 @@ checkDollarQuoteParen params t = where fix id = fixWith [replaceStart id params 2 "\"$"] +prop_checkTranslatedString1 = verify checkTranslatedString "foo_bar2=val; $\"foo_bar2\"" +prop_checkTranslatedString2 = verifyNot checkTranslatedString "$\"foo_bar2\"" +prop_checkTranslatedString3 = verifyNot checkTranslatedString "$\"..\"" +prop_checkTranslatedString4 = verifyNot checkTranslatedString "var=val; $\"$var\"" +prop_checkTranslatedString5 = verifyNot checkTranslatedString "foo=var; bar=val2; $\"foo bar\"" +checkTranslatedString params (T_DollarDoubleQuoted id ((T_Literal _ s):_)) = + fromMaybe (return ()) $ do + Map.lookup s assignments + return $ + warnWithFix id 2256 "This translated string is the name of a variable. Flip leading $ and \" if this should be a quoted substitution." (fix id) + where + assignments = foldl (flip ($)) Map.empty (map insertAssignment $ variableFlow params) + insertAssignment (Assignment (_, token, name, _)) | isVariableName name = + Map.insert name token + insertAssignment _ = Prelude.id + fix id = fixWith [replaceStart id params 2 "\"$"] +checkTranslatedString _ _ = return () + prop_checkDefaultCase1 = verify checkDefaultCase "case $1 in a) true ;; esac" prop_checkDefaultCase2 = verify checkDefaultCase "case $1 in ?*?) true ;; *? ) true ;; esac" prop_checkDefaultCase3 = verifyNot checkDefaultCase "case $1 in x|*) true ;; esac" From 4a63a3a8bdd7ed7525f44226c094570da3786e94 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 13 Nov 2019 19:53:55 -0800 Subject: [PATCH 208/763] For SC2256, make sure the complete string is a variable name --- src/ShellCheck/Analytics.hs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index b0928ce..a29d76d 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -192,7 +192,7 @@ nodeChecks = [ ,checkRedirectionToCommand ,checkDollarQuoteParen ,checkUselessBang - ,checkTranslatedString + ,checkTranslatedStringVariable ] optionalChecks = map fst optionalTreeChecks @@ -3452,13 +3452,14 @@ checkDollarQuoteParen params t = where fix id = fixWith [replaceStart id params 2 "\"$"] -prop_checkTranslatedString1 = verify checkTranslatedString "foo_bar2=val; $\"foo_bar2\"" -prop_checkTranslatedString2 = verifyNot checkTranslatedString "$\"foo_bar2\"" -prop_checkTranslatedString3 = verifyNot checkTranslatedString "$\"..\"" -prop_checkTranslatedString4 = verifyNot checkTranslatedString "var=val; $\"$var\"" -prop_checkTranslatedString5 = verifyNot checkTranslatedString "foo=var; bar=val2; $\"foo bar\"" -checkTranslatedString params (T_DollarDoubleQuoted id ((T_Literal _ s):_)) = +prop_checkTranslatedStringVariable1 = verify checkTranslatedStringVariable "foo_bar2=val; $\"foo_bar2\"" +prop_checkTranslatedStringVariable2 = verifyNot checkTranslatedStringVariable "$\"foo_bar2\"" +prop_checkTranslatedStringVariable3 = verifyNot checkTranslatedStringVariable "$\"..\"" +prop_checkTranslatedStringVariable4 = verifyNot checkTranslatedStringVariable "var=val; $\"$var\"" +prop_checkTranslatedStringVariable5 = verifyNot checkTranslatedStringVariable "foo=var; bar=val2; $\"foo bar\"" +checkTranslatedStringVariable params (T_DollarDoubleQuoted id [T_Literal _ s]) = fromMaybe (return ()) $ do + guard $ all isVariableChar s Map.lookup s assignments return $ warnWithFix id 2256 "This translated string is the name of a variable. Flip leading $ and \" if this should be a quoted substitution." (fix id) @@ -3468,7 +3469,7 @@ checkTranslatedString params (T_DollarDoubleQuoted id ((T_Literal _ s):_)) = Map.insert name token insertAssignment _ = Prelude.id fix id = fixWith [replaceStart id params 2 "\"$"] -checkTranslatedString _ _ = return () +checkTranslatedStringVariable _ _ = return () prop_checkDefaultCase1 = verify checkDefaultCase "case $1 in a) true ;; esac" prop_checkDefaultCase2 = verify checkDefaultCase "case $1 in ?*?) true ;; *? ) true ;; esac" From c75bbcbd60383b4b20ed516760dac3be01f0c872 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 13 Nov 2019 22:09:52 -0800 Subject: [PATCH 209/763] Include missing Semigroup import --- src/ShellCheck/Analytics.hs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index a29d76d..866d621 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -41,6 +41,7 @@ import Data.Function (on) import Data.List import Data.Maybe import Data.Ord +import Data.Semigroup import Debug.Trace import qualified Data.Map.Strict as Map import Test.QuickCheck.All (forAllProperties) From f44624a9c0b6cb84eab355493ef7d421c2b365d2 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 14 Nov 2019 20:02:25 -0800 Subject: [PATCH 210/763] Hide <> from Writer to not conflict with Semigroup --- src/ShellCheck/Analytics.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 866d621..d8bfcd4 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -33,7 +33,7 @@ import Control.Arrow (first) import Control.Monad import Control.Monad.Identity import Control.Monad.State -import Control.Monad.Writer +import Control.Monad.Writer hiding ((<>)) import Control.Monad.Reader import Data.Char import Data.Functor From 9b1befadc113fd7dbc6ddde601ea1a4acfa99b27 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Fri, 15 Nov 2019 09:26:01 -0800 Subject: [PATCH 211/763] Update brew before building on macOS due to incompatible Ruby --- .compile_binaries | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.compile_binaries b/.compile_binaries index 2585c14..44ddc58 100755 --- a/.compile_binaries +++ b/.compile_binaries @@ -56,7 +56,8 @@ build_windows() { } build_osx() { - # Darwin x86_64 static executable + # Darwin x86_64 executable + brew update brew install cabal-install pandoc gnu-tar sudo ln -s /usr/local/bin/gsha512sum /usr/local/bin/sha512sum sudo ln -s /usr/local/bin/gtar /usr/local/bin/tar From 2c026f1ec7c205c731ff2a0ccd85365f37245758 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 16 Nov 2019 10:44:48 -0800 Subject: [PATCH 212/763] Support Cabal 3. Man page no longer autobuilds. --- Dockerfile | 4 ++-- Setup.hs | 36 ------------------------------------ ShellCheck.cabal | 12 +++--------- manpage | 4 ++++ 4 files changed, 9 insertions(+), 47 deletions(-) delete mode 100644 Setup.hs create mode 100755 manpage diff --git a/Dockerfile b/Dockerfile index 6ca370e..671b9ea 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build-only image -FROM ubuntu:18.04 AS build +FROM ubuntu:19.10 AS build USER root WORKDIR /opt/shellCheck @@ -12,7 +12,7 @@ COPY ShellCheck.cabal ./ RUN cabal update && cabal install --dependencies-only --ghc-options="-optlo-Os -split-sections" # Copy source and build it -COPY LICENSE Setup.hs shellcheck.hs ./ +COPY LICENSE shellcheck.hs ./ COPY src src RUN cabal build Paths_ShellCheck && \ ghc -optl-static -optl-pthread -isrc -idist/build/autogen --make shellcheck -split-sections -optc-Wl,--gc-sections -optlo-Os && \ diff --git a/Setup.hs b/Setup.hs deleted file mode 100644 index a909cf6..0000000 --- a/Setup.hs +++ /dev/null @@ -1,36 +0,0 @@ -import Distribution.PackageDescription ( - HookedBuildInfo, - emptyHookedBuildInfo ) -import Distribution.Simple ( - Args, - UserHooks ( preSDist ), - defaultMainWithHooks, - simpleUserHooks ) -import Distribution.Simple.Setup ( SDistFlags ) - -import System.Process ( system ) - - -main = defaultMainWithHooks myHooks - where - myHooks = simpleUserHooks { preSDist = myPreSDist } - --- | This hook will be executed before e.g. @cabal sdist@. It runs --- pandoc to create the man page from shellcheck.1.md. If the pandoc --- command is not found, this will fail with an error message: --- --- /bin/sh: pandoc: command not found --- --- Since the man page is listed in the Extra-Source-Files section of --- our cabal file, a failure here should result in a failure to --- create the distribution tarball (that's a good thing). --- -myPreSDist :: Args -> SDistFlags -> IO HookedBuildInfo -myPreSDist _ _ = do - putStrLn "Building the man page (shellcheck.1) with pandoc..." - putStrLn pandoc_cmd - result <- system pandoc_cmd - putStrLn $ "pandoc exited with " ++ show result - return emptyHookedBuildInfo - where - pandoc_cmd = "pandoc -s -f markdown-smart -t man shellcheck.1.md -o shellcheck.1" diff --git a/ShellCheck.cabal b/ShellCheck.cabal index 372bde4..0389be6 100644 --- a/ShellCheck.cabal +++ b/ShellCheck.cabal @@ -7,7 +7,7 @@ Category: Static Analysis Author: Vidar Holen Maintainer: vidar@vidarholen.net Homepage: https://www.shellcheck.net/ -Build-Type: Custom +Build-Type: Simple Cabal-Version: >= 1.8 Bug-reports: https://github.com/koalaman/shellcheck/issues Description: @@ -26,19 +26,13 @@ Extra-Source-Files: -- documentation README.md shellcheck.1.md - -- built with a cabal sdist hook - shellcheck.1 + -- A script to build the man page using pandoc + manpage -- convenience script for stripping tests striptests -- tests test/shellcheck.hs -custom-setup - setup-depends: - base >= 4 && <5, - process >= 1.0 && <1.7, - Cabal >= 1.10 && <2.5 - source-repository head type: git location: git://github.com/koalaman/shellcheck.git diff --git a/manpage b/manpage new file mode 100755 index 0000000..27967f5 --- /dev/null +++ b/manpage @@ -0,0 +1,4 @@ +#!/bin/sh +echo >&2 "Generating man page using pandoc" +pandoc -s -f markdown-smart -t man shellcheck.1.md -o shellcheck.1 || exit +echo >&2 "Done. You can read it with: man ./shellcheck.1" From 9f578f41a1259b67f45976fe4eb61366239661b0 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 16 Nov 2019 11:16:15 -0800 Subject: [PATCH 213/763] Explicitly add 'mappend' for old GHC versions --- src/ShellCheck/Analytics.hs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index d8bfcd4..cc5a285 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1824,6 +1824,7 @@ instance Semigroup SpaceStatus where (x, SpaceEmpty) -> x instance Monoid SpaceStatus where mempty = SpaceEmpty + mappend = (<>) -- This is slightly awkward because we want to support structured -- optional checks based on nearly the same logic From e075cde35792523140aa0506abc0423758151944 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 16 Nov 2019 11:46:58 -0800 Subject: [PATCH 214/763] Revert docker image to 18.04 since ld fails on later versions --- Dockerfile | 2 +- manpage | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 671b9ea..0ce807d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build-only image -FROM ubuntu:19.10 AS build +FROM ubuntu:18.04 AS build USER root WORKDIR /opt/shellCheck diff --git a/manpage b/manpage index 27967f5..0898092 100755 --- a/manpage +++ b/manpage @@ -1,4 +1,4 @@ #!/bin/sh -echo >&2 "Generating man page using pandoc" +echo >&2 "Generating man page using pandoc" pandoc -s -f markdown-smart -t man shellcheck.1.md -o shellcheck.1 || exit echo >&2 "Done. You can read it with: man ./shellcheck.1" From 5c7d8129ad3af8b99902dd656a0ffef3be1141d5 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 18 Nov 2019 17:12:25 -0800 Subject: [PATCH 215/763] Try to search for binary on macOS/Cabal3 --- .compile_binaries | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.compile_binaries b/.compile_binaries index 44ddc58..4ec71de 100755 --- a/.compile_binaries +++ b/.compile_binaries @@ -66,9 +66,14 @@ build_osx() { cabal update cabal install --dependencies-only cabal build shellcheck + + # Cabal 3 no longer has a predictable output path + path="$(find . -name 'shellcheck' -type f -perm +111)" + [[ -e "$path" ]] + for tag in $TAGS do - cp "dist/build/shellcheck/shellcheck" "deploy/shellcheck-$tag.darwin-x86_64"; + cp "$path" "deploy/shellcheck-$tag.darwin-x86_64"; done } From 0a4580e23473003602a40301ffa6a093582d5d6f Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 7 Dec 2019 16:08:44 -0800 Subject: [PATCH 216/763] Mention that ShellCheck is now compatible with Cabal 3 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d7e4181..6f7840b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ### Fixed - `-f diff` no longer claims that it found more issues when it didn't - Known empty variables now correctly trigger SC2086 +- ShellCheck should now be compatible with Cabal 3 ### Added - SC2254: Suggest quoting expansions in case statements From 0f15fa49ba8cc427aaad338140f82707b6865eb5 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 7 Dec 2019 16:06:34 -0800 Subject: [PATCH 217/763] Make SC2230 optional --- CHANGELOG.md | 3 +++ shellcheck.1.md | 4 ++-- src/ShellCheck/Analyzer.hs | 9 +++++---- src/ShellCheck/Checks/Commands.hs | 31 +++++++++++++++++++++++++++---- 4 files changed, 37 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f7840b..fc8736a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ - SC2255: Suggest using `$((..))` in `[ 2*3 -eq 6 ]` - SC2256: Warn about translated strings that are known variables +### Changed +- SC2230: This check is now off by default + ## v0.7.0 - 2019-07-28 ### Added - Precompiled binaries for macOS and Linux aarch64 diff --git a/shellcheck.1.md b/shellcheck.1.md index 02175dc..187d12a 100644 --- a/shellcheck.1.md +++ b/shellcheck.1.md @@ -275,8 +275,8 @@ Here is an example `.shellcheckrc`: # Turn on warnings for unassigned uppercase variables enable=check-unassigned-uppercase - # Allow using `which` since it gives full paths and is common enough - disable=SC2230 + # Allow [ ! -z foo ] instead of suggesting -n + disable=SC2236 If no `.shellcheckrc` is found in any of the parent directories, ShellCheck will look in `~/.shellcheckrc` followed by the XDG config directory diff --git a/src/ShellCheck/Analyzer.hs b/src/ShellCheck/Analyzer.hs index 33d2ae0..eb231c2 100644 --- a/src/ShellCheck/Analyzer.hs +++ b/src/ShellCheck/Analyzer.hs @@ -35,17 +35,18 @@ analyzeScript spec = newAnalysisResult { arComments = filterByAnnotation spec params . nub $ runAnalytics spec - ++ runChecker params (checkers params) + ++ runChecker params (checkers spec params) } where params = makeParameters spec -checkers params = mconcat $ map ($ params) [ - ShellCheck.Checks.Commands.checker, +checkers spec params = mconcat $ map ($ params) [ + ShellCheck.Checks.Commands.checker spec, ShellCheck.Checks.Custom.checker, ShellCheck.Checks.ShellSupport.checker ] optionalChecks = mconcat $ [ - ShellCheck.Analytics.optionalChecks + ShellCheck.Analytics.optionalChecks, + ShellCheck.Checks.Commands.optionalChecks ] diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 7f06862..5481204 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -21,7 +21,7 @@ {-# LANGUAGE FlexibleContexts #-} -- This module contains checks that examine specific commands by name. -module ShellCheck.Checks.Commands (checker , ShellCheck.Checks.Commands.runTests) where +module ShellCheck.Checks.Commands (checker, optionalChecks, ShellCheck.Checks.Commands.runTests) where import ShellCheck.AST import ShellCheck.ASTLib @@ -90,13 +90,30 @@ commandChecks = [ ,checkMvArguments, checkCpArguments, checkLnArguments ,checkFindRedirections ,checkReadExpansions - ,checkWhich ,checkSudoRedirect ,checkSudoArgs ,checkSourceArgs ,checkChmodDashr ] +optionalChecks = map fst optionalCommandChecks +optionalCommandChecks :: [(CheckDescription, CommandCheck)] +optionalCommandChecks = [ + (newCheckDescription { + cdName = "deprecate-which", + cdDescription = "Suggest 'command -v' instead of 'which'", + cdPositive = "which javac", + cdNegative = "command -v javac" + }, checkWhich) + ] +optionalCheckMap = Map.fromList $ map (\(desc, check) -> (cdName desc, check)) optionalCommandChecks + +prop_verifyOptionalExamples = all check optionalCommandChecks + where + check (desc, check) = + verify check (cdPositive desc) + && verifyNot check (cdNegative desc) + buildCommandMap :: [CommandCheck] -> Map.Map CommandName (Token -> Analysis) buildCommandMap = foldl' addCheck Map.empty where @@ -128,8 +145,14 @@ getChecker list = Checker { map = buildCommandMap list -checker :: Parameters -> Checker -checker params = getChecker commandChecks +checker :: AnalysisSpec -> Parameters -> Checker +checker spec params = getChecker $ commandChecks ++ optionals + where + keys = asOptionalChecks spec + optionals = + if "all" `elem` keys + then map snd optionalCommandChecks + else mapMaybe (\x -> Map.lookup x optionalCheckMap) keys prop_checkTr1 = verify checkTr "tr [a-f] [A-F]" prop_checkTr2 = verify checkTr "tr 'a-z' 'A-Z'" From 3f296a08c19d518ee824915e2b13b6bda3fc703a Mon Sep 17 00:00:00 2001 From: Gandalf- Date: Wed, 18 Dec 2019 20:23:48 -0800 Subject: [PATCH 218/763] Issue 1731 Literals in case patterns https://github.com/koalaman/shellcheck/issues/1731 Any literal except esac is valid pattern in a case statement --- src/ShellCheck/Parser.hs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 849fbd7..42ec9c1 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -1040,14 +1040,16 @@ prop_readNormalWord9 = isOk readSubshell "(foo\\ ;\nbar)" prop_readNormalWord10 = isWarning readNormalWord "\x201Chello\x201D" prop_readNormalWord11 = isWarning readNormalWord "\x2018hello\x2019" prop_readNormalWord12 = isWarning readNormalWord "hello\x2018" -readNormalWord = readNormalishWord "" +readNormalWord = readNormalishWord "" ["do", "done", "then", "fi", "esac"] -readNormalishWord end = do +readPatternWord = readNormalishWord "" ["esac"] + +readNormalishWord end terms = do start <- startSpan pos <- getPosition x <- many1 (readNormalWordPart end) id <- endSpan start - checkPossibleTermination pos x + checkPossibleTermination pos x terms return $ T_NormalWord id x readIndexSpan = do @@ -1067,10 +1069,10 @@ readIndexSpan = do id <- endSpan start return $ T_Literal id str -checkPossibleTermination pos [T_Literal _ x] = - when (x `elem` ["do", "done", "then", "fi", "esac"]) $ +checkPossibleTermination pos [T_Literal _ x] terminators = + when (x `elem` terminators) $ parseProblemAt pos WarningC 1010 $ "Use semicolon or linefeed before '" ++ x ++ "' (or quote to make it literal)." -checkPossibleTermination _ _ = return () +checkPossibleTermination _ _ _ = return () readNormalWordPart end = do notFollowedBy2 $ oneOf end @@ -2655,7 +2657,7 @@ readCoProc = called "coproc" $ do return $ T_CoProcBody id body -readPattern = (readNormalWord `thenSkip` spacing) `sepBy1` (char '|' `thenSkip` spacing) +readPattern = (readPatternWord `thenSkip` spacing) `sepBy1` (char '|' `thenSkip` spacing) prop_readCompoundCommand = isOk readCompoundCommand "{ echo foo; }>/dev/null" readCompoundCommand = do From 83187dafd7b7b0547a2d335b4cdfaf244f7a8ccb Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 21 Dec 2019 17:59:09 -0800 Subject: [PATCH 219/763] Added a unit test for parsing shell keyword case branches --- src/ShellCheck/Parser.hs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 42ec9c1..6a0e6b8 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -2534,6 +2534,7 @@ prop_readCaseClause2 = isOk readCaseClause "case foo\n in * ) echo bar;; esac" prop_readCaseClause3 = isOk readCaseClause "case foo\n in * ) echo bar & ;; esac" prop_readCaseClause4 = isOk readCaseClause "case foo\n in *) echo bar ;& bar) foo; esac" prop_readCaseClause5 = isOk readCaseClause "case foo\n in *) echo bar;;& foo) baz;; esac" +prop_readCaseClause6 = isOk readCaseClause "case foo\n in if) :;; done) :;; esac" readCaseClause = called "case expression" $ do start <- startSpan g_Case From fdd02c94c01b81b78e2dada903ea88e29a39befe Mon Sep 17 00:00:00 2001 From: Gandalf- Date: Sun, 22 Dec 2019 23:11:20 -0800 Subject: [PATCH 220/763] Issue 1759 mapfile and process substition https://github.com/koalaman/shellcheck/issues/1759 When a simple process substition is used, this tripped up the getMapfileArray function by making the last argument not a variable --- src/ShellCheck/Analytics.hs | 1 + src/ShellCheck/AnalyzerLib.hs | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 582b8cd..d6ea460 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2190,6 +2190,7 @@ prop_checkUnassignedReferences36= verifyNotTree checkUnassignedReferences "read prop_checkUnassignedReferences37= verifyNotTree checkUnassignedReferences "var=howdy; printf -v 'array[0]' %s \"$var\"; printf %s \"${array[0]}\";" prop_checkUnassignedReferences38= verifyTree (checkUnassignedReferences' True) "echo $VAR" prop_checkUnassignedReferences39= verifyNotTree checkUnassignedReferences "builtin export var=4; echo $var" +prop_checkUnassignedReferences40= verifyNotTree checkUnassignedReferences "mapfile -t files <(cat); echo \"${files[@]}\"" checkUnassignedReferences = checkUnassignedReferences' False checkUnassignedReferences' includeGlobals params t = warnings diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 590889c..1e30c7a 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -670,13 +670,17 @@ getModifiedVariableCommand base@(T_SimpleCommand id cmdPrefix (T_NormalWord _ (T -- mapfile has some curious syntax allowing flags plus 0..n variable names -- where only the first non-option one is used if any. Here we cheat and - -- just get the last one, if it's a variable name. + -- just get the last one, if it's a variable name, and omitting process + -- substitions. getMapfileArray base arguments = do - lastArg <- listToMaybe (reverse arguments) + lastArg <- listToMaybe (filter notProcSub $ reverse arguments) name <- getLiteralString lastArg guard $ isVariableName name return (base, lastArg, name, DataArray SourceExternal) + notProcSub (T_NormalWord _ (T_ProcSub{} :_)) = False + notProcSub _ = True + -- get all the array variables used in read, e.g. read -a arr getReadArrayVariables args = map (getLiteralArray . snd) From 926ee54036f7922829355027d9be4029b0831e3a Mon Sep 17 00:00:00 2001 From: Artur Klauser Date: Sat, 28 Dec 2019 09:51:11 +0100 Subject: [PATCH 221/763] Fix OSX build on Travis Symlink was failing due to pre-existing target file: sudo ln -s /usr/local/bin/gsha512sum /usr/local/bin/sha512sum ln: /usr/local/bin/sha512sum: File exists --- .compile_binaries | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.compile_binaries b/.compile_binaries index 4ec71de..b535ae9 100755 --- a/.compile_binaries +++ b/.compile_binaries @@ -59,8 +59,8 @@ build_osx() { # Darwin x86_64 executable brew update brew install cabal-install pandoc gnu-tar - sudo ln -s /usr/local/bin/gsha512sum /usr/local/bin/sha512sum - sudo ln -s /usr/local/bin/gtar /usr/local/bin/tar + sudo ln -sf /usr/local/bin/gsha512sum /usr/local/bin/sha512sum + sudo ln -sf /usr/local/bin/gtar /usr/local/bin/tar export PATH="/usr/local/bin:$PATH" cabal update From b96b7f35f45565c960c904f3c02d2730d6a1e543 Mon Sep 17 00:00:00 2001 From: Artur Klauser Date: Sat, 28 Dec 2019 09:29:05 +0100 Subject: [PATCH 222/763] Fix Travis warnings Fixing the following Travis build config validation warnings: * root: deprecated key sudo (The key `sudo` has no effect anymore.) * root: missing os, using the default linux * deploy: key local-dir is not underscored, using local_dir * language: value sh is an alias for shell, using shell * root: key matrix is an alias for jobs, using jobs --- .travis.yml | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6f658a6..43f0a35 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,23 +1,17 @@ - -sudo: required - -language: sh +language: shell +os: linux services: - docker -matrix: +jobs: include: - - os: linux - env: BUILD=linux - - os: linux - env: BUILD=windows - - os: linux - env: BUILD=armv6hf - - os: linux - env: BUILD=aarch64 - - os: osx - env: BUILD=osx + - env: BUILD=linux + - env: BUILD=windows + - env: BUILD=armv6hf + - env: BUILD=aarch64 + - env: BUILD=osx + os: osx before_install: | DOCKER_BASE="$DOCKER_USERNAME/shellcheck" @@ -60,7 +54,7 @@ deploy: secret_access_key: secure: Bcx2cT0/E2ikj7sdamVq52xlLZF9dz9ojGPtoKfPyQhkkZa+McVI4xgUSuyyoSxyKj77sofx2y8m6PJYYumT4g5hREV1tfeUkl0J2DQFMbGDYEt7kxVkXCxojNvhHwTzLFv0ezstrxWWxQm81BfQQ4U9lggRXtndAP4czZnOeHPINPSiue1QNwRAEw05r5UoIUJXy/5xyUrjIxn381pAs+gJqP2COeN9kTKYH53nS/AAws29RprfZFnPlo7xxWmcjRcdS5KPdGXI/c6tQp5zl2iTh510VC1PN2w1Wvnn/oNWhiNdqPyVDsojIX5+sS3nejzJA+KFMxXSBlyXIY3wPpS/MdscU79X6Q5f9ivsFfsm7gNBmxHUPNn0HAvU4ROT/CCE9j6jSbs5PC7QBo3CK4++jxAwE/pd9HUc2rs3k0ofx3rgveJ7txpy5yPKfwIIBi98kVKlC4w7dLvNTOfjW1Imt2yH87XTfsE0UIG9st1WII6s4l/WgBx2GuwKdt6+3QUYiAlCFckkxWi+fAvpHZUEL43Qxub5fN+ZV7Zib1n7opchH4QKGBb6/y0WaDCmtCfu0lppoe/TH6saOTjDFj67NJSElK6ZDxGZ3uw4R+ret2gm6WRKT2Oeub8J33VzSa7VkmFpMPrAAfPa9N1Z4ewBLoTmvxSg2A0dDrCdJio= bucket: shellcheck - local-dir: deploy + local_dir: deploy on: repo: koalaman/shellcheck all_branches: true From 499e0ceaba063c35bf3739e2495b5ec2ab493871 Mon Sep 17 00:00:00 2001 From: Artur Klauser Date: Fri, 27 Dec 2019 07:49:06 +0100 Subject: [PATCH 223/763] Add multi-architecture Docker image build * Adds a shell script with functions to install multi-architecture docker support, as well as build, deploy, and test the shellcheck docker images for the same set of architectures for which binaries were already built and deployed as tarballs. * Hooks up the multi-architecture docker build, deploy, and test to the existing Travis CI/CD pipeline. It is organized as a separate stage which only runs if all previous steps in the already existing test stage succeed. --- .compile_binaries | 6 --- .multi_arch_docker | 107 ++++++++++++++++++++++++++++++++++++++++++ .travis.yml | 21 ++++----- Dockerfile | 7 --- Dockerfile.multi-arch | 26 ++++++++++ 5 files changed, 141 insertions(+), 26 deletions(-) create mode 100755 .multi_arch_docker create mode 100644 Dockerfile.multi-arch diff --git a/.compile_binaries b/.compile_binaries index b535ae9..f657d76 100755 --- a/.compile_binaries +++ b/.compile_binaries @@ -19,12 +19,6 @@ build_linux() { do cp "shellcheck" "deploy/shellcheck-$tag.linux-x86_64"; done - - # Linux Alpine based Docker image - name="$DOCKER_BASE-alpine" - DOCKER_BUILDS="$DOCKER_BUILDS $name" - docker build -f Dockerfile -t "$name:current" --target alpine . - docker run "$name:current" sh -c 'shellcheck --version' } build_aarch64() { diff --git a/.multi_arch_docker b/.multi_arch_docker new file mode 100755 index 0000000..93c0859 --- /dev/null +++ b/.multi_arch_docker @@ -0,0 +1,107 @@ +#!/bin/bash +# This script builds and deploys multi-architecture docker images from the +# binaries previously built and deployed to GCS by the Travis pipeline. + +function multi_arch_docker::install_docker_buildx() { + # Install up-to-date version of docker, with buildx support. + local -r docker_apt_repo='https://download.docker.com/linux/ubuntu' + curl -fsSL "${docker_apt_repo}/gpg" | sudo apt-key add - + local -r os="$(lsb_release -cs)" + sudo add-apt-repository "deb [arch=amd64] $docker_apt_repo $os stable" + sudo apt-get update + sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce + + # Enable docker daemon experimental support (for 'pull --platform'). + local -r config='/etc/docker/daemon.json' + if [[ -e "$config" ]]; then + sudo sed -i -e 's/{/{ "experimental": true, /' "$config" + else + echo '{ "experimental": true }' | sudo tee "$config" + fi + sudo systemctl restart docker + + # Install QEMU multi-architecture support for docker buildx. + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + + # Instantiate docker buildx builder with multi-architecture support. + export DOCKER_CLI_EXPERIMENTAL=enabled + docker buildx create --name mybuilder + docker buildx use mybuilder + # Start up buildx and verify that all is OK. + docker buildx inspect --bootstrap +} + +# Log in to Docker Hub for deployment. +function multi_arch_docker::login_to_docker_hub() { + echo "$DOCKER_PASSWORD" | docker login -u="$DOCKER_USERNAME" --password-stdin +} + +# Run buildx build and push. Passed in arguments augment the command line. +function multi_arch_docker::buildx() { + mkdir -p /tmp/empty + docker buildx build \ + --platform "${DOCKER_PLATFORMS// /,}" \ + --push \ + --progress plain \ + -f Dockerfile.multi-arch \ + "$@" \ + /tmp/empty + rmdir /tmp/empty +} + +# Build and push plain and alpine docker images for all tags. +function multi_arch_docker::build_and_push_all() { + for tag in $TAGS; do + multi_arch_docker::buildx -t "$DOCKER_BASE:$tag" --build-arg "tag=$tag" + multi_arch_docker::buildx -t "$DOCKER_BASE-alpine:$tag" \ + --build-arg "tag=$tag" --target alpine + done +} + +# Test all pushed docker images. +function multi_arch_docker::test_all() { + printf '%s\n' "#!/bin/sh" "echo 'hello world'" > myscript + + for platform in $DOCKER_PLATFORMS; do + for tag in $TAGS; do + for ext in '-alpine' ''; do + image="${DOCKER_BASE}${ext}:${tag}" + msg="Testing docker image $image on platform $platform" + line="${msg//?/=}" + printf '\n%s\n%s\n%s\n' "${line}" "${msg}" "${line}" + docker pull -q --platform "$platform" "$image" + if [ -n "$ext" ]; then + echo -n "Image architecture: " + docker run --rm --entrypoint /bin/sh "$image" -c 'uname -m' + version=$(docker run --rm "$image" shellcheck --version \ + | grep 'version:') + else + version=$(docker run --rm "$image" --version | grep 'version:') + fi + version=${version/#version: /v} + echo "shellcheck version: $version" + if [[ ! ("$tag" =~ ^(latest|stable)$) && "$tag" != "$version" ]]; then + echo "Version mismatch: shellcheck $version tagged as $tag" + exit 1 + fi + if [ -n "$ext" ]; then + docker run --rm -v "$PWD:/mnt" -w /mnt "$image" shellcheck myscript + else + docker run --rm -v "$PWD:/mnt" "$image" myscript + fi + done + done + done +} + +function multi_arch_docker::main() { + export DOCKER_PLATFORMS='linux/amd64' + DOCKER_PLATFORMS+=' linux/arm64' + DOCKER_PLATFORMS+=' linux/arm/v6' + + multi_arch_docker::install_docker_buildx + multi_arch_docker::login_to_docker_hub + multi_arch_docker::build_and_push_all + set +x + multi_arch_docker::test_all +} diff --git a/.travis.yml b/.travis.yml index 43f0a35..6c601a6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,13 +6,19 @@ services: jobs: include: - - env: BUILD=linux + - stage: Test + env: BUILD=linux - env: BUILD=windows - env: BUILD=armv6hf - env: BUILD=aarch64 - env: BUILD=osx os: osx + - stage: Deploy docker image + script: + - source ./.multi_arch_docker + - set -ex; multi_arch_docker::main; set +x + before_install: | DOCKER_BASE="$DOCKER_USERNAME/shellcheck" DOCKER_BUILDS="" @@ -28,18 +34,6 @@ script: - set -ex; build_"$BUILD"; set +x; - ./.prepare_deploy -after_success: | - if [ "$BUILD" = "linux" ]; then - docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD" - for repo in $DOCKER_BUILDS; do - for tag in $TAGS; do - echo "Deploying $repo:current as $repo:$tag..."; - docker tag "$repo:current" "$repo:$tag" || exit 1; - docker push "$repo:$tag" || exit 1; - done; - done; - fi - after_failure: | id pwd @@ -57,4 +51,5 @@ deploy: local_dir: deploy on: repo: koalaman/shellcheck + condition: $TRAVIS_BUILD_STAGE_NAME = Test all_branches: true diff --git a/Dockerfile b/Dockerfile index 0ce807d..2f8f79e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,13 +21,6 @@ RUN cabal build Paths_ShellCheck && \ RUN mkdir -p /out/bin && \ cp shellcheck /out/bin/ -# Resulting Alpine image -FROM alpine:latest AS alpine -LABEL maintainer="Vidar Holen " -COPY --from=build /out / - -# DELETE-MARKER (Remove everything below to keep the alpine image) - # Resulting ShellCheck image FROM scratch LABEL maintainer="Vidar Holen " diff --git a/Dockerfile.multi-arch b/Dockerfile.multi-arch new file mode 100644 index 0000000..193e762 --- /dev/null +++ b/Dockerfile.multi-arch @@ -0,0 +1,26 @@ +# Alpine image +FROM alpine:latest AS alpine +LABEL maintainer="Vidar Holen " +ARG tag + +# Put the right binary for each architecture into place for the +# multi-architecture docker image. +RUN set -x; \ + arch="$(uname -m)"; \ + echo "arch is $arch"; \ + if [ "${arch}" = 'armv7l' ]; then \ + arch='armv6hf'; \ + fi; \ + url_base='https://shellcheck.storage.googleapis.com/'; \ + tar_file="shellcheck-${tag}.linux.${arch}.tar.xz"; \ + wget "${url_base}${tar_file}" -O - | tar xJf -; \ + mv "shellcheck-${tag}/shellcheck" /bin/; \ + rm -rf "shellcheck-${tag}"; \ + ls -laF /bin/shellcheck + +# ShellCheck image +FROM scratch +LABEL maintainer="Vidar Holen " +WORKDIR /mnt +COPY --from=alpine /bin/shellcheck /bin/ +ENTRYPOINT ["/bin/shellcheck"] From 93486ed6aca215e46eed064c4458e902fe31e8c5 Mon Sep 17 00:00:00 2001 From: Marcin Szydelski Date: Tue, 21 Jan 2020 12:43:27 +0100 Subject: [PATCH 224/763] SC2016: disable for mumps -run %XCMD and LOOP%XCMD --- src/ShellCheck/Analytics.hs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 582b8cd..c758d36 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -934,6 +934,11 @@ prop_checkSingleQuotedVariables16= verify checkSingleQuotedVariables "git '$a'" prop_checkSingleQuotedVariables17= verifyNot checkSingleQuotedVariables "rename 's/(.)a/$1/g' *" prop_checkSingleQuotedVariables18= verifyNot checkSingleQuotedVariables "echo '``'" prop_checkSingleQuotedVariables19= verifyNot checkSingleQuotedVariables "echo '```'" +prop_checkSingleQuotedVariables20= verifyNot checkSingleQuotedVariables "mumps -run %XCMD 'W $O(^GLOBAL(5))'" +prop_checkSingleQuotedVariables21= verifyNot checkSingleQuotedVariables "mumps -run LOOP%XCMD --xec 'W $O(^GLOBAL(6))'" + + + checkSingleQuotedVariables params t@(T_SingleQuoted id s) = when (s `matches` re) $ @@ -947,7 +952,7 @@ checkSingleQuotedVariables params t@(T_SingleQuoted id s) = commandName = fromMaybe "" $ do cmd <- getClosestCommand parents t name <- getCommandBasename cmd - return $ if name == "find" then getFindCommand cmd else if name == "git" then getGitCommand cmd else name + return $ if name == "find" then getFindCommand cmd else if name == "git" then getGitCommand cmd else if name == "mumps" then getMumpsCommand cmd else name isProbablyOk = any isOkAssignment (take 3 $ getPath parents t) @@ -968,6 +973,8 @@ checkSingleQuotedVariables params t@(T_SingleQuoted id s) = ,"rename" ,"unset" ,"git filter-branch" + ,"mumps -run %XCMD" + ,"mumps -run LOOP%XCMD" ] || "awk" `isSuffixOf` commandName || "perl" `isPrefixOf` commandName @@ -997,6 +1004,13 @@ checkSingleQuotedVariables params t@(T_SingleQuoted id s) = _ -> "git" getGitCommand (T_Redirecting _ _ cmd) = getGitCommand cmd getGitCommand _ = "git" + getMumpsCommand (T_SimpleCommand _ _ words) = + case map getLiteralString words of + Just "mumps":Just "-run":Just "%XCMD":_ -> "mumps -run %XCMD" + Just "mumps":Just "-run":Just "LOOP%XCMD":_ -> "mumps -run LOOP%XCMD" + _ -> "mumps" + getMumpsCommand (T_Redirecting _ _ cmd) = getMumpsCommand cmd + getMumpsCommand _ = "mumps" checkSingleQuotedVariables _ _ = return () From a82e606e8d18e03ff18072a064b42cc8413c0fc6 Mon Sep 17 00:00:00 2001 From: Peter Gromov Date: Fri, 31 Jan 2020 14:49:25 +0100 Subject: [PATCH 225/763] Don't trigger SC2154 (unassigned var) in `-n`/`-z` expressions #1583 --- src/ShellCheck/Analytics.hs | 10 ++++++++++ src/ShellCheck/AnalyzerLib.hs | 11 ++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 582b8cd..cf952f5 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2191,6 +2191,16 @@ prop_checkUnassignedReferences37= verifyNotTree checkUnassignedReferences "var=h prop_checkUnassignedReferences38= verifyTree (checkUnassignedReferences' True) "echo $VAR" prop_checkUnassignedReferences39= verifyNotTree checkUnassignedReferences "builtin export var=4; echo $var" +prop_checkUnassignedReferences_minusNPlain = verifyNotTree checkUnassignedReferences "if [ -n \"$x\" ]; then echo $x; fi" +prop_checkUnassignedReferences_minusZPlain = verifyNotTree checkUnassignedReferences "if [ -z \"$x\" ]; then echo \"\"; fi" +prop_checkUnassignedReferences_minusNBraced = verifyNotTree checkUnassignedReferences "if [ -n \"${x}\" ]; then echo $x; fi" +prop_checkUnassignedReferences_minusZBraced = verifyNotTree checkUnassignedReferences "if [ -z \"${x}\" ]; then echo \"\"; fi" +prop_checkUnassignedReferences_minusNDefault = verifyNotTree checkUnassignedReferences "if [ -n \"${x:-}\" ]; then echo $x; fi" +prop_checkUnassignedReferences_minusZDefault = verifyNotTree checkUnassignedReferences "if [ -z \"${x:-}\" ]; then echo \"\"; fi" + +prop_checkUnassignedReferences_minusZInsteadOfN = verifyTree checkUnassignedReferences "if [ -z \"$x\" ]; then echo $x; fi" +prop_checkUnassignedReferences_minusZInsteadOfNBraced = verifyTree checkUnassignedReferences "if [ -z \"${x}\" ]; then echo $x; fi" + checkUnassignedReferences = checkUnassignedReferences' False checkUnassignedReferences' includeGlobals params t = warnings where diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 590889c..b5e740a 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -510,6 +510,9 @@ getModifiedVariables t = guard . not . null $ str return (t, token, str, DataString SourceChecked) + TC_Unary _ _ "-n" (T_NormalWord _ [T_DoubleQuoted _ [db@(T_DollarBraced _ _ _)]]) -> + [(t, t, getBracedReference (bracedString db), DataString SourceChecked)] + T_DollarBraced _ _ l -> maybeToList $ do let string = bracedString t let modifier = getBracedModifier string @@ -719,7 +722,9 @@ getOffsetReferences mods = fromMaybe [] $ do getReferencedVariables parents t = case t of T_DollarBraced id _ l -> let str = bracedString t in - (t, t, getBracedReference str) : + if isMinusZTest t + then [] + else (t, t, getBracedReference str) : map (\x -> (l, l, x)) ( getIndexReferences str ++ getOffsetReferences (getBracedModifier str)) @@ -774,6 +779,10 @@ getReferencedVariables parents t = this: TA_Assignment _ "=" lhs _ :_ -> lhs == t _ -> False + isMinusZTest t = case getPath parents t of + _ : T_DoubleQuoted _ [_] : T_NormalWord _ [_] : TC_Unary _ SingleBracket "-z" _ : _ -> True + _ -> False + dataTypeFrom defaultType v = (case v of T_Array {} -> DataArray; _ -> defaultType) $ SourceFrom [v] From 1696296c0ad25cdad59b5d0e7cfd51ba14633a13 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 1 Feb 2020 16:51:40 -0800 Subject: [PATCH 226/763] Make SC2141 trigger more broadly --- src/ShellCheck/Analytics.hs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 582b8cd..acd62b1 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2651,21 +2651,25 @@ checkMultipleAppends params t = prop_checkSuspiciousIFS1 = verify checkSuspiciousIFS "IFS=\"\\n\"" prop_checkSuspiciousIFS2 = verifyNot checkSuspiciousIFS "IFS=$'\\t'" -checkSuspiciousIFS params (T_Assignment id Assign "IFS" [] value) = +prop_checkSuspiciousIFS3 = verify checkSuspiciousIFS "IFS=' \\t\\n'" +checkSuspiciousIFS params (T_Assignment _ _ "IFS" [] value) = potentially $ do str <- getLiteralString value return $ check str where - n = if shellType params == Sh then "''" else "$'\\n'" - t = if shellType params == Sh then "\"$(printf '\\t')\"" else "$'\\t'" + hasDollarSingle = shellType params == Bash || shellType params == Ksh + n = if hasDollarSingle then "$'\\n'" else "''" + t = if hasDollarSingle then "$'\\t'" else "\"$(printf '\\t')\"" check value = case value of "\\n" -> suggest n - "/n" -> suggest n "\\t" -> suggest t - "/t" -> suggest t + x | '\\' `elem` x -> suggest2 "a literal backslash" + x | 'n' `elem` x -> suggest2 "the literal letter 'n'" + x | 't' `elem` x -> suggest2 "the literal letter 't'" _ -> return () - suggest r = warn id 2141 $ "Did you mean IFS=" ++ r ++ " ?" + suggest r = warn (getId value) 2141 $ "This backslash is literal. Did you mean IFS=" ++ r ++ " ?" + suggest2 desc = warn (getId value) 2141 $ "This IFS value contains " ++ desc ++ ". For tabs/linefeeds/escapes, use $'..', literal, or printf." checkSuspiciousIFS _ _ = return () From 2e52c2b56a2bccb2f06c90a081e27f0fe35cf814 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sat, 1 Feb 2020 22:50:11 -0500 Subject: [PATCH 227/763] Use notElem instead of not on the result of elem --- src/ShellCheck/Analytics.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 582b8cd..64a71a3 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -3169,7 +3169,7 @@ checkSplittingInArrays params t = T_DollarBraced id _ str | not (isCountingReference part) && not (isQuotedAlternativeReference part) - && not (getBracedReference (bracedString part) `elem` variablesWithoutSpaces) + && getBracedReference (bracedString part) `notElem` variablesWithoutSpaces -> warn id 2206 $ if shellType params == Ksh then "Quote to prevent word splitting/globbing, or split robustly with read -A or while read." From 3449e6be215dc3d2ec05032dcfc282897a49e836 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sat, 1 Feb 2020 22:50:13 -0500 Subject: [PATCH 228/763] Get rid of our getOpt, as it already exists as lookup --- src/ShellCheck/Analytics.hs | 2 +- src/ShellCheck/AnalyzerLib.hs | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 64a71a3..e8bee4f 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2840,7 +2840,7 @@ checkReadWithoutR _ t@T_SimpleCommand {} | t `isUnqualifiedCommand` "read" = flags = getAllFlags t has_t0 = fromMaybe False $ do parsed <- getOpts flagsForRead flags - t <- getOpt "t" parsed + t <- lookup "t" parsed str <- getLiteralString t return $ str == "0" diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 590889c..e93758f 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -960,8 +960,6 @@ getOpts string flags = process flags more <- process rest2 return $ (flag1, token1) : more -getOpt str flags = snd <$> (listToMaybe $ filter (\(f, _) -> f == str) $ flags) - supportsArrays shell = shell == Bash || shell == Ksh -- Returns true if the shell is Bash or Ksh (sorry for the name, Ksh) From 93be86f988e17b6228a23924966da3d6e827bc1b Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sat, 1 Feb 2020 22:50:14 -0500 Subject: [PATCH 229/763] Use "drop 1" instead of clumsily rewriting it --- src/ShellCheck/AnalyzerLib.hs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index e93758f..563f8ea 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -612,8 +612,7 @@ getModifiedVariableCommand base@(T_SimpleCommand id cmdPrefix (T_NormalWord _ (T _ -> [] where flags = map snd $ getAllFlags base - stripEquals s = let rest = dropWhile (/= '=') s in - if rest == "" then "" else tail rest + stripEquals s = drop 1 $ dropWhile (/= '=') s stripEqualsFrom (T_NormalWord id1 (T_Literal id2 s:rs)) = T_NormalWord id1 (T_Literal id2 (stripEquals s):rs) stripEqualsFrom (T_NormalWord id1 [T_DoubleQuoted id2 [T_Literal id3 s]]) = From 0f48bb78a50d4b76b2d1838ee22e3f00d5e8bf1f Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sat, 1 Feb 2020 22:50:14 -0500 Subject: [PATCH 230/763] Remove incorrect otherwise You're supposed to use otherwise where you need a Boolean, not a pattern match. This is misleadingly shadowing the real otherwise. Use _ instead. --- src/ShellCheck/Checks/ShellSupport.hs | 2 +- src/ShellCheck/Data.hs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 07dfdda..e340d41 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -487,7 +487,7 @@ checkBraceExpansionVars = ForShell [Bash] f T_DollarBraced {} -> return "$" T_DollarExpansion {} -> return "$" T_DollarArithmetic {} -> return "$" - otherwise -> return "-" + _ -> return "-" toString t = fromJust $ getLiteralStringExt literalExt t isEvaled t = do cmd <- getClosestCommandM t diff --git a/src/ShellCheck/Data.hs b/src/ShellCheck/Data.hs index 732619d..fb4a1e4 100644 --- a/src/ShellCheck/Data.hs +++ b/src/ShellCheck/Data.hs @@ -135,6 +135,6 @@ shellForExecutable name = "ksh" -> return Ksh "ksh88" -> return Ksh "ksh93" -> return Ksh - otherwise -> Nothing + _ -> Nothing flagsForRead = "sreu:n:N:i:p:a:t:" From f5c6771016e396b8a0c11f8f2c6e58cb967c8356 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sat, 1 Feb 2020 22:50:16 -0500 Subject: [PATCH 231/763] Use find instead of listToMaybe and filter --- src/ShellCheck/Analytics.hs | 6 +++--- src/ShellCheck/Checks/ShellSupport.hs | 4 ++-- src/ShellCheck/Parser.hs | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index e8bee4f..e33d873 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1234,10 +1234,10 @@ checkLiteralBreakingTest _ t = potentially $ return () comparisonWarning list = do - token <- listToMaybe $ filter hasEquals list + token <- find hasEquals list return $ err (getId token) 2077 "You need spaces around the comparison operator." tautologyWarning t s = do - token <- listToMaybe $ filter isNonEmpty $ getWordParts t + token <- find isNonEmpty $ getWordParts t return $ err (getId token) 2157 s prop_checkConstantNullary = verify checkConstantNullary "[[ '$(foo)' ]]" @@ -2910,7 +2910,7 @@ checkLoopVariableReassignment params token = where check = do str <- loopVariable token - next <- listToMaybe $ filter (\x -> loopVariable x == Just str) path + next <- find (\x -> loopVariable x == Just str) path return $ do warn (getId token) 2165 "This nested loop overrides the index variable of its parent." warn (getId next) 2167 "This parent loop has its index variable overridden." diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index e340d41..742cfa5 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -340,8 +340,8 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do potentially $ do allowed' <- Map.lookup name allowedFlags allowed <- allowed' - (word, flag) <- listToMaybe $ - filter (\x -> (not . null . snd $ x) && snd x `notElem` allowed) flags + (word, flag) <- find + (\x -> (not . null . snd $ x) && snd x `notElem` allowed) flags return . warnMsg (getId word) $ name ++ " -" ++ flag ++ " is" when (name == "source") $ warnMsg id "'source' in place of '.' is" diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 339b50b..f295560 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -34,7 +34,7 @@ import Control.Monad.Identity import Control.Monad.Trans import Data.Char import Data.Functor -import Data.List (isPrefixOf, isInfixOf, isSuffixOf, partition, sortBy, intercalate, nub) +import Data.List (isPrefixOf, isInfixOf, isSuffixOf, partition, sortBy, intercalate, nub, find) import Data.Maybe import Data.Monoid import Debug.Trace @@ -589,7 +589,7 @@ readConditionContents single = checkTrailingOp x = fromMaybe (return ()) $ do (T_Literal id str) <- getTrailingUnquotedLiteral x - trailingOp <- listToMaybe (filter (`isSuffixOf` str) binaryTestOps) + trailingOp <- find (`isSuffixOf` str) binaryTestOps return $ parseProblemAtId id ErrorC 1108 $ "You need a space before and after the " ++ trailingOp ++ " ." From 28978a8b65b303143b2251bc286527b3095c3622 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sat, 1 Feb 2020 22:50:17 -0500 Subject: [PATCH 232/763] Use maybe instead of fromMaybe and fmap --- src/ShellCheck/Analytics.hs | 10 +++++----- src/ShellCheck/AnalyzerLib.hs | 4 ++-- src/ShellCheck/Checker.hs | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index e33d873..8d10a02 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -409,7 +409,7 @@ prop_checkArithmeticOpCommand1 = verify checkArithmeticOpCommand "i=i + 1" prop_checkArithmeticOpCommand2 = verify checkArithmeticOpCommand "foo=bar * 2" prop_checkArithmeticOpCommand3 = verifyNot checkArithmeticOpCommand "foo + opts" checkArithmeticOpCommand _ (T_SimpleCommand id [T_Assignment {}] (firstWord:_)) = - fromMaybe (return ()) $ check <$> getGlobOrLiteralString firstWord + maybe (return ()) check $ getGlobOrLiteralString firstWord where check op = when (op `elem` ["+", "-", "*", "/"]) $ @@ -493,8 +493,8 @@ checkPipePitfalls _ (T_Pipeline id _ commands) = do for ["grep", "wc"] $ \(grep:wc:_) -> - let flagsGrep = fromMaybe [] $ map snd . getAllFlags <$> getCommand grep - flagsWc = fromMaybe [] $ map snd . getAllFlags <$> getCommand wc + let flagsGrep = maybe [] (map snd . getAllFlags) $ getCommand grep + flagsWc = maybe [] (map snd . getAllFlags) $ getCommand wc in unless (any (`elem` ["o", "only-matching", "r", "R", "recursive"]) flagsGrep || any (`elem` ["m", "chars", "w", "words", "c", "bytes", "L", "max-line-length"]) flagsWc || null flagsWc) $ style (getId grep) 2126 "Consider using grep -c instead of grep|wc -l." @@ -3140,9 +3140,9 @@ checkSubshellAsTest _ t = checkParams id first second = do - when (fromMaybe False $ (`elem` unaryTestOps) <$> getLiteralString first) $ + when (maybe False (`elem` unaryTestOps) $ getLiteralString first) $ err id 2204 "(..) is a subshell. Did you mean [ .. ], a test expression?" - when (fromMaybe False $ (`elem` binaryTestOps) <$> getLiteralString second) $ + when (maybe False (`elem` binaryTestOps) $ getLiteralString second) $ warn id 2205 "(..) is a subshell. Did you mean [ .. ], a test expression?" diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 563f8ea..4dd3f9d 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -784,8 +784,8 @@ isCommand token str = isCommandMatch token (\cmd -> cmd == str || ('/' : str) ` -- Compare a command to a literal. Like above, but checks full path. isUnqualifiedCommand token str = isCommandMatch token (== str) -isCommandMatch token matcher = fromMaybe False $ - fmap matcher (getCommandName token) +isCommandMatch token matcher = maybe False + matcher (getCommandName token) -- Does this regex look like it was intended as a glob? -- True: *foo* diff --git a/src/ShellCheck/Checker.hs b/src/ShellCheck/Checker.hs index 2ea950d..2837d9a 100644 --- a/src/ShellCheck/Checker.hs +++ b/src/ShellCheck/Checker.hs @@ -88,9 +88,9 @@ checkScript sys spec = do asOptionalChecks = csOptionalChecks spec } where as = newAnalysisSpec root let analysisMessages = - fromMaybe [] $ + maybe [] (arComments . analyzeScript . analysisSpec) - <$> prRoot result + $ prRoot result let translator = tokenToPosition tokenPositions return . nub . sortMessages . filter shouldInclude $ (parseMessages ++ map translator analysisMessages) From 5487b3f229fa3ce56f105733caff6a707c39e7bb Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sat, 1 Feb 2020 22:50:18 -0500 Subject: [PATCH 233/763] Use sortOn instead of sortBy and comparing --- src/ShellCheck/Checker.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Checker.hs b/src/ShellCheck/Checker.hs index 2837d9a..b423f2d 100644 --- a/src/ShellCheck/Checker.hs +++ b/src/ShellCheck/Checker.hs @@ -104,7 +104,7 @@ checkScript sys spec = do code = cCode (pcComment pc) severity = cSeverity (pcComment pc) - sortMessages = sortBy (comparing order) + sortMessages = sortOn order order pc = let pos = pcStartPos pc comment = pcComment pc in From d7278b95f2c840a5a764838d23a32008aaeb6168 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sat, 1 Feb 2020 22:50:19 -0500 Subject: [PATCH 234/763] Remove unnecessary "map snd" --- src/ShellCheck/Checks/Commands.hs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index eb0c434..7841089 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -997,8 +997,7 @@ missingDestination handler token = do args = getAllFlags token params = map fst $ filter (\(_,x) -> x == "") args hasTarget = - any (\x -> x /= "" && x `isPrefixOf` "target-directory") $ - map snd args + any (\(_,x) -> x /= "" && x `isPrefixOf` "target-directory") args prop_checkMvArguments1 = verify checkMvArguments "mv 'foo bar'" prop_checkMvArguments2 = verifyNot checkMvArguments "mv foo bar" From f25b8bd03af4997df3572973a09da025f417422c Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sat, 1 Feb 2020 22:50:20 -0500 Subject: [PATCH 235/763] Use gets instead of fmapping the result of get --- src/ShellCheck/Analytics.hs | 2 +- src/ShellCheck/Parser.hs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 8d10a02..ae8e81d 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1961,7 +1961,7 @@ prop_checkQuotesInLiterals9 = verifyNotTree checkQuotesInLiterals "param=\"/foo/ checkQuotesInLiterals params t = doVariableFlowAnalysis readF writeF Map.empty (variableFlow params) where - getQuotes name = fmap (Map.lookup name) get + getQuotes name = gets (Map.lookup name) setQuotes name ref = modify $ Map.insert name ref deleteQuotes = modify . Map.delete parents = parentMap params diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index f295560..fa9084e 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -325,7 +325,7 @@ parseProblem level code msg = do parseProblemAt pos level code msg setCurrentContexts c = Ms.modify (\state -> state { contextStack = c }) -getCurrentContexts = contextStack <$> Ms.get +getCurrentContexts = Ms.gets contextStack popContext = do v <- getCurrentContexts From e6e89d68fdda2fb1f8f7ec2e4485ab7a5584db15 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sat, 1 Feb 2020 22:50:20 -0500 Subject: [PATCH 236/763] Use list comprehensions instead of clunky combinations of map and filter --- src/ShellCheck/Checks/Commands.hs | 6 +++--- src/ShellCheck/Fixer.hs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 7841089..21c6cb7 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -706,7 +706,7 @@ checkReadExpansions = CommandCheck (Exactly "read") check options = getGnuOpts flagsForRead getVars cmd = fromMaybe [] $ do opts <- options cmd - return . map snd $ filter (\(x,_) -> x == "" || x == "a") opts + return [y | (x,y) <- opts, x == "" || x == "a"] check cmd = mapM_ warning $ getVars cmd warning t = potentially $ do @@ -995,7 +995,7 @@ missingDestination handler token = do _ -> return () where args = getAllFlags token - params = map fst $ filter (\(_,x) -> x == "") args + params = [x | (x,"") <- args] hasTarget = any (\(_,x) -> x /= "" && x `isPrefixOf` "target-directory") args @@ -1083,7 +1083,7 @@ checkSudoArgs = CommandCheck (Basename "sudo") f where f t = potentially $ do opts <- parseOpts t - let nonFlags = map snd $ filter (\(flag, _) -> flag == "") opts + let nonFlags = [x | ("",x) <- opts] commandArg <- nonFlags !!! 0 command <- getLiteralString commandArg guard $ command `elem` builtins diff --git a/src/ShellCheck/Fixer.hs b/src/ShellCheck/Fixer.hs index 12de3c2..a69616e 100644 --- a/src/ShellCheck/Fixer.hs +++ b/src/ShellCheck/Fixer.hs @@ -295,7 +295,7 @@ prop_pstreeSumsCorrectly kvs targets = -- Trivial O(n * m) implementation dumbPrefixSums :: [(Int, Int)] -> [Int] -> [Int] dumbPrefixSums kvs targets = - let prefixSum target = sum . map snd . filter (\(k,v) -> k <= target) $ kvs + let prefixSum target = sum [v | (k,v) <- kvs, k <= target] in map prefixSum targets -- PSTree O(n * log m) implementation smartPrefixSums :: [(Int, Int)] -> [Int] -> [Int] From c29b6afa5600d68ff3dced9259ff0b5f59822864 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sat, 1 Feb 2020 22:50:21 -0500 Subject: [PATCH 237/763] Use null instead of comparing with empty lists --- src/ShellCheck/Analytics.hs | 4 ++-- src/ShellCheck/AnalyzerLib.hs | 4 ++-- src/ShellCheck/Checker.hs | 12 ++++++------ src/ShellCheck/Checks/Commands.hs | 6 +++--- src/ShellCheck/Parser.hs | 2 +- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index ae8e81d..2888047 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -563,7 +563,7 @@ checkShebang params (T_Annotation _ list t) = isOverride _ = False checkShebang params (T_Script _ (T_Literal id sb) _) = execWriter $ do unless (shellTypeSpecified params) $ do - when (sb == "") $ + when (null sb) $ err id 2148 "Tips depend on target shell and yours is unknown. Add a shebang." when (executableFromShebang sb == "ash") $ warn id 2187 "Ash scripts will be checked as Dash. Add '# shellcheck shell=dash' to silence." @@ -2332,7 +2332,7 @@ checkWhileReadPitfalls _ (T_WhileExpression id [command] contents) checkMuncher _ = return () stdinRedirect (T_FdRedirect _ fd _) - | fd == "" || fd == "0" = True + | null fd || fd == "0" = True stdinRedirect _ = False checkWhileReadPitfalls _ _ = return () diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 4dd3f9d..e4640e7 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -643,7 +643,7 @@ getModifiedVariableCommand base@(T_SimpleCommand id cmdPrefix (T_NormalWord _ (T getModifierParam _ _ = [] letParamToLiteral token = - if var == "" + if null var then [] else [(base, token, var, DataString $ SourceFrom [stripEqualsFrom token])] where var = takeWhile isVariableChar $ dropWhile (`elem` "+-") $ concat $ oversimplify token @@ -952,7 +952,7 @@ getOpts string flags = process flags takesArg <- Map.lookup flag1 flagMap if takesArg then do - guard $ flag2 == "" + guard $ null flag2 more <- process rest return $ (flag1, token2) : more else do diff --git a/src/ShellCheck/Checker.hs b/src/ShellCheck/Checker.hs index b423f2d..6370f75 100644 --- a/src/ShellCheck/Checker.hs +++ b/src/ShellCheck/Checker.hs @@ -198,11 +198,11 @@ prop_optionDisablesBadShebang = } prop_annotationDisablesBadShebang = - [] == check "#!/usr/bin/python\n# shellcheck shell=sh\ntrue\n" + null $ check "#!/usr/bin/python\n# shellcheck shell=sh\ntrue\n" prop_canParseDevNull = - [] == check "source /dev/null" + null $ check "source /dev/null" prop_failsWhenNotSourcing = [1091, 2154] == check "source lol; echo \"$bar\"" @@ -218,7 +218,7 @@ prop_worksWhenDotting = -- FIXME: This should really be giving [1093], "recursively sourced" prop_noInfiniteSourcing = - [] == checkWithIncludes [("lib", "source lib")] "source lib" + null $ checkWithIncludes [("lib", "source lib")] "source lib" prop_canSourceBadSyntax = [1094, 2086] == checkWithIncludes [("lib", "for f; do")] "source lib; echo $1" @@ -239,10 +239,10 @@ prop_recursiveParsing = [1037] == checkRecursive [("lib", "echo \"$10\"")] "source lib" prop_nonRecursiveAnalysis = - [] == checkWithIncludes [("lib", "echo $1")] "source lib" + null $ checkWithIncludes [("lib", "echo $1")] "source lib" prop_nonRecursiveParsing = - [] == checkWithIncludes [("lib", "echo \"$10\"")] "source lib" + null $ checkWithIncludes [("lib", "echo \"$10\"")] "source lib" prop_sourceDirectiveDoesntFollowFile = null $ checkWithIncludes @@ -328,7 +328,7 @@ prop_optionIncludes4 = [2154] == checkOptionIncludes (Just [2154]) "#!/bin/sh\n var='a b'\n echo $var\n echo $bar" -prop_readsRcFile = result == [] +prop_readsRcFile = null result where result = checkWithRc "disable=2086" emptyCheckSpec { csScript = "#!/bin/sh\necho $1", diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 21c6cb7..299a335 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -345,7 +345,7 @@ returnOrExit multi invalid = (f . arguments) invalid (getId value) f _ = return () - isInvalid s = s == "" || any (not . isDigit) s || length s > 5 + isInvalid s = null s || any (not . isDigit) s || length s > 5 || let value = (read s :: Integer) in value > 255 literal token = fromJust $ getLiteralStringExt lit token @@ -706,7 +706,7 @@ checkReadExpansions = CommandCheck (Exactly "read") check options = getGnuOpts flagsForRead getVars cmd = fromMaybe [] $ do opts <- options cmd - return [y | (x,y) <- opts, x == "" || x == "a"] + return [y | (x,y) <- opts, null x || x == "a"] check cmd = mapM_ warning $ getVars cmd warning t = potentially $ do @@ -1057,7 +1057,7 @@ checkSudoRedirect = CommandCheck (Basename "sudo") f Just (T_Redirecting _ redirs _) -> mapM_ warnAbout redirs warnAbout (T_FdRedirect _ s (T_IoFile id op file)) - | (s == "" || s == "&") && not (special file) = + | (null s || s == "&") && not (special file) = case op of T_Less _ -> info (getId op) 2024 diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index fa9084e..727572d 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -3169,7 +3169,7 @@ readScriptFile sourced = do Nothing -> parseProblemAt pos ErrorC 1008 "This shebang was unrecognized. ShellCheck only supports sh/bash/dash/ksh. Add a 'shell' directive to specify." isValidShell s = - let good = s == "" || any (`isPrefixOf` s) goodShells + let good = null s || any (`isPrefixOf` s) goodShells bad = any (`isPrefixOf` s) badShells in if good From 8a005526cc5b1379b0c651a0505f7110c6bf0d8e Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sat, 1 Feb 2020 23:02:10 -0500 Subject: [PATCH 238/763] Use drop instead of splitAt since we only use the second half --- src/ShellCheck/Fixer.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Fixer.hs b/src/ShellCheck/Fixer.hs index a69616e..16bdd98 100644 --- a/src/ShellCheck/Fixer.hs +++ b/src/ShellCheck/Fixer.hs @@ -200,7 +200,7 @@ doReplace start end o r = let si = fromIntegral (start-1) ei = fromIntegral (end-1) (x, xs) = splitAt si o - (y, z) = splitAt (ei - si) xs + z = drop (ei - si) xs in x ++ r ++ z From 76b798394f18307677b44e231d6922db251955be Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sat, 1 Feb 2020 23:07:16 -0500 Subject: [PATCH 239/763] Use case matching instead of null Using null followed by a head, tail, or a partial pattern match is an anti-pattern. Use case matching instead. --- src/ShellCheck/Parser.hs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 727572d..025fa98 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -329,12 +329,11 @@ getCurrentContexts = Ms.gets contextStack popContext = do v <- getCurrentContexts - if not $ null v - then do - let (a:r) = v + case v of + (a:r) -> do setCurrentContexts r return $ Just a - else + [] -> return Nothing pushContext c = do From 115ef290798d8d4b046d2a57470c461ce3b26d6f Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 2 Feb 2020 00:13:16 -0500 Subject: [PATCH 240/763] Use pattern matching instead of head --- src/ShellCheck/Analytics.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 2888047..cf8015b 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1298,7 +1298,7 @@ checkArithmeticDeref params t@(TA_Expansion _ [b@(T_DollarBraced id _ _)]) = unless (isException $ bracedString b) getWarning where isException [] = True - isException s = any (`elem` "/.:#%?*@$-!+=^,") s || isDigit (head s) + isException s@(h:_) = any (`elem` "/.:#%?*@$-!+=^,") s || isDigit h getWarning = fromMaybe noWarning . msum . map warningFor $ parents params t warningFor t = case t of From 6595e14d25deed617b4c8a9c7d2162ee29d77198 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 2 Feb 2020 00:22:52 -0500 Subject: [PATCH 241/763] Adjust a pattern to avoid tail --- src/ShellCheck/Analytics.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index cf8015b..ddc3b43 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1644,9 +1644,9 @@ checkSpuriousExec _ = doLists doList = doList' . stripCleanup -- The second parameter is True if we are in a loop -- In that case we should emit the warning also if `exec' is the last statement - doList' t@(current:following:_) False = do + doList' (current:t@(following:_)) False = do commentIfExec current - doList (tail t) False + doList t False doList' (current:tail) True = do commentIfExec current doList tail True From 392b57b8e848a3008a0851d3212a4657870230a1 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 2 Feb 2020 00:27:05 -0500 Subject: [PATCH 242/763] Use maybe instead of isJust and fromJust --- src/ShellCheck/Checks/ShellSupport.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 742cfa5..b643525 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -491,7 +491,7 @@ checkBraceExpansionVars = ForShell [Bash] f toString t = fromJust $ getLiteralStringExt literalExt t isEvaled t = do cmd <- getClosestCommandM t - return $ isJust cmd && fromJust cmd `isUnqualifiedCommand` "eval" + return $ maybe False (`isUnqualifiedCommand` "eval") cmd prop_checkMultiDimensionalArrays1 = verify checkMultiDimensionalArrays "foo[a][b]=3" From e820a5642b4029e3bfe793f59b4cd23dd22b7c20 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 2 Feb 2020 00:34:54 -0500 Subject: [PATCH 243/763] Adjust a pattern to get rid of a fromJust --- src/ShellCheck/Analytics.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index ddc3b43..501847a 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2635,8 +2635,8 @@ checkMultipleAppends params t = where checkList list = mapM_ checkGroup (groupWith (fmap fst) $ map getTarget list) - checkGroup (f:_:_:_) | isJust f = - style (snd $ fromJust f) 2129 + checkGroup (Just (_,id):_:_:_) = + style id 2129 "Consider using { cmd1; cmd2; } >> file instead of individual redirects." checkGroup _ = return () getTarget (T_Annotation _ _ t) = getTarget t From fe2b4b5079ce986467d02ad4209963b7c9799bac Mon Sep 17 00:00:00 2001 From: Furkan Pham Date: Mon, 3 Feb 2020 13:00:10 +0100 Subject: [PATCH 244/763] Fix pre-compiled binary URL for aarch64 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f31867b..5acfb89 100644 --- a/README.md +++ b/README.md @@ -219,7 +219,7 @@ Alternatively, you can download pre-compiled binaries for the latest release her * [Linux, x86_64](https://storage.googleapis.com/shellcheck/shellcheck-stable.linux.x86_64.tar.xz) (statically linked) * [Linux, armv6hf](https://storage.googleapis.com/shellcheck/shellcheck-stable.linux.armv6hf.tar.xz), i.e. Raspberry Pi (statically linked) -* [Linux, aarch64](https://storage.googleapis.com/shellcheck/shellcheck-stable.linux.armv6hf.tar.xz) aka ARM64 (statically linked) +* [Linux, aarch64](https://storage.googleapis.com/shellcheck/shellcheck-stable.linux.aarch64.tar.xz) aka ARM64 (statically linked) * [MacOS, x86_64](https://shellcheck.storage.googleapis.com/shellcheck-stable.darwin.x86_64.tar.xz) * [Windows, x86](https://storage.googleapis.com/shellcheck/shellcheck-stable.zip) From 474b23d6e726adeddd946e250030e0ac06f512a4 Mon Sep 17 00:00:00 2001 From: Benjamin Gordon Date: Wed, 5 Feb 2020 16:25:18 -0700 Subject: [PATCH 245/763] SC2191: Tighten index checks When adding a value containing an equals sign to an indexed array, the left side is treated as an index if it looks like [N]=val and N is numeric. SC2191 currently warns about anything that looks like key=val even though non-numeric values of key will never be treated as an index. This causes spurious warnings for the common pattern of building up program arguments in an array, such as: args=( --dry-run --in="${my_var}" --out=/some/path -f ) /bin/program "${args[@]}" Since only numeric expressions can be a valid index for an indexed array, only emit SC2191 if the left side of a literal string containing an equals looks numeric. Other more complicated constructs should still warn because shellcheck doesn't know if they may evaluate to a numeric result. Associative arrays still warn because a non-numeric left side is a valid subscript. --- src/ShellCheck/Analytics.hs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index acd62b1..d04d15e 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -3025,9 +3025,13 @@ prop_checkArrayAssignmentIndices3 = verifyNotTree checkArrayAssignmentIndices "d prop_checkArrayAssignmentIndices4 = verifyTree checkArrayAssignmentIndices "typeset -A foo; foo+=(bar)" prop_checkArrayAssignmentIndices5 = verifyTree checkArrayAssignmentIndices "arr=( [foo]= bar )" prop_checkArrayAssignmentIndices6 = verifyTree checkArrayAssignmentIndices "arr=( [foo] = bar )" -prop_checkArrayAssignmentIndices7 = verifyTree checkArrayAssignmentIndices "arr=( var=value )" +prop_checkArrayAssignmentIndices7 = verifyNotTree checkArrayAssignmentIndices "arr=( var=value )" prop_checkArrayAssignmentIndices8 = verifyNotTree checkArrayAssignmentIndices "arr=( [foo]=bar )" prop_checkArrayAssignmentIndices9 = verifyNotTree checkArrayAssignmentIndices "arr=( [foo]=\"\" )" +prop_checkArrayAssignmentIndices10 = verifyTree checkArrayAssignmentIndices "declare -A arr; arr=( var=value )" +prop_checkArrayAssignmentIndices11 = verifyTree checkArrayAssignmentIndices "arr=( 1=value )" +prop_checkArrayAssignmentIndices12 = verifyTree checkArrayAssignmentIndices "arr=( $a=value )" +prop_checkArrayAssignmentIndices13 = verifyTree checkArrayAssignmentIndices "arr=( $((1+1))=value )" checkArrayAssignmentIndices params root = runNodeAnalysis check params root where @@ -3052,7 +3056,7 @@ checkArrayAssignmentIndices params root = (id, str) <- case part of T_Literal id str -> [(id,str)] _ -> [] - guard $ '=' `elem` str + guard $ '=' `elem` str && hasNumericIndex str return $ warnWithFix id 2191 "The = here is literal. To assign by index, use ( [index]=value ) with no spaces. To keep as literal, quote it." (surroundWidth id params "\"") in if null literalEquals && isAssociative @@ -3060,6 +3064,9 @@ checkArrayAssignmentIndices params root = else sequence_ literalEquals _ -> return () + where + hasNumericIndex str = all isDigit $ takeWhile (/= '=') str + prop_checkUnmatchableCases1 = verify checkUnmatchableCases "case foo in bar) true; esac" prop_checkUnmatchableCases2 = verify checkUnmatchableCases "case foo-$bar in ??|*) true; esac" From ef51ed3950a0a516d6fe149b0c0183e0bad941d4 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sat, 8 Feb 2020 14:09:17 -0500 Subject: [PATCH 246/763] Simplify literalEquals --- src/ShellCheck/Analytics.hs | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 8a2902c..d0db0f7 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -3052,11 +3052,9 @@ checkArrayAssignmentIndices params root = T_NormalWord _ parts -> let literalEquals = do - part <- parts - (id, str) <- case part of - T_Literal id str -> [(id,str)] - _ -> [] - guard $ '=' `elem` str && hasNumericIndex str + T_Literal id str <- parts + let (before, after) = break ('=' ==) str + guard $ all isDigit before && not (null after) return $ warnWithFix id 2191 "The = here is literal. To assign by index, use ( [index]=value ) with no spaces. To keep as literal, quote it." (surroundWidth id params "\"") in if null literalEquals && isAssociative @@ -3064,8 +3062,6 @@ checkArrayAssignmentIndices params root = else sequence_ literalEquals _ -> return () - where - hasNumericIndex str = all isDigit $ takeWhile (/= '=') str prop_checkUnmatchableCases1 = verify checkUnmatchableCases "case foo in bar) true; esac" From bd116f252b1dba4d07f3a10994fc00529158ad91 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sat, 8 Feb 2020 22:55:45 -0500 Subject: [PATCH 247/763] Use findM instead of filterM Using filterM makes us run the monadic predicate on every list element. Use findM instead so that we can stop once we find a matching one. --- shellcheck.hs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/shellcheck.hs b/shellcheck.hs index 20fb4b6..f1757ef 100644 --- a/shellcheck.hs +++ b/shellcheck.hs @@ -491,6 +491,12 @@ ioInterface options files = do first <- a arg if not first then return False else b arg + findM p = foldr go (pure Nothing) + where + go x acc = do + b <- p x + if b then pure (Just x) else acc + findSourceFile inputs sourcePathFlag currentScript sourcePathAnnotation original = if isAbsolute original then @@ -500,11 +506,11 @@ ioInterface options files = do find original original where find filename deflt = do - sources <- filterM ((allowable inputs) `andM` doesFileExist) $ + sources <- findM ((allowable inputs) `andM` doesFileExist) $ (adjustPath filename):(map ( filename) $ map adjustPath $ sourcePathFlag ++ sourcePathAnnotation) case sources of - [] -> return deflt - (first:_) -> return first + Nothing -> return deflt + Just first -> return first scriptdir = dropFileName currentScript adjustPath str = case (splitDirectories str) of From aaffe381987cd253ae8dac4b77af08fca2d7982d Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sat, 8 Feb 2020 23:06:57 -0500 Subject: [PATCH 248/763] Use the Identity monad to avoid unnecessary uses of fromJust --- src/ShellCheck/ASTLib.hs | 5 +++-- src/ShellCheck/AnalyzerLib.hs | 2 +- src/ShellCheck/Checks/Commands.hs | 7 ++++--- src/ShellCheck/Checks/ShellSupport.hs | 3 ++- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 03a2f9a..658ed64 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -25,6 +25,7 @@ import Control.Monad.Writer import Control.Monad import Data.Char import Data.Functor +import Data.Functor.Identity import Data.List import Data.Maybe @@ -177,7 +178,7 @@ getLiteralString = getLiteralStringExt (const Nothing) -- Definitely get a literal string, skipping over all non-literals onlyLiteralString :: Token -> String -onlyLiteralString = fromJust . getLiteralStringExt (const $ return "") +onlyLiteralString = runIdentity . getLiteralStringExt (const $ return "") -- Maybe get a literal string, but only if it's an unquoted argument. getUnquotedLiteral (T_NormalWord _ list) = @@ -216,7 +217,7 @@ getGlobOrLiteralString = getLiteralStringExt f -- Maybe get the literal value of a token, using a custom function -- to map unrecognized Tokens into strings. -getLiteralStringExt :: (Token -> Maybe String) -> Token -> Maybe String +getLiteralStringExt :: Monad m => (Token -> m String) -> Token -> m String getLiteralStringExt more = g where allInList = fmap concat . mapM g diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index e4640e7..0da80e3 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -806,7 +806,7 @@ isVariableName (x:r) = isVariableStartChar x && all isVariableChar r isVariableName _ = False getVariablesFromLiteralToken token = - getVariablesFromLiteral (fromJust $ getLiteralStringExt (const $ return " ") token) + getVariablesFromLiteral (runIdentity $ getLiteralStringExt (const $ return " ") token) -- Try to get referenced variables from a literal string like "$foo" -- Ignores tons of cases like arithmetic evaluation and array indices. diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 299a335..869b389 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -34,6 +34,7 @@ import ShellCheck.Regex import Control.Monad import Control.Monad.RWS import Data.Char +import Data.Functor.Identity import Data.List import Data.Maybe import qualified Data.Map.Strict as Map @@ -348,7 +349,7 @@ returnOrExit multi invalid = (f . arguments) isInvalid s = null s || any (not . isDigit) s || length s > 5 || let value = (read s :: Integer) in value > 255 - literal token = fromJust $ getLiteralStringExt lit token + literal token = runIdentity $ getLiteralStringExt lit token lit (T_DollarBraced {}) = return "0" lit (T_DollarArithmetic {}) = return "0" lit (T_DollarExpansion {}) = return "0" @@ -735,7 +736,7 @@ checkAliasesUsesArgs = CommandCheck (Exactly "alias") (f . arguments) re = mkRegex "\\$\\{?[0-9*@]" f = mapM_ checkArg checkArg arg = - let string = fromJust $ getLiteralStringExt (const $ return "_") arg in + let string = runIdentity $ getLiteralStringExt (const $ return "_") arg in when ('=' `elem` string && string `matches` re) $ err (getId arg) 2142 "Aliases can't use positional parameters. Use a function." @@ -781,7 +782,7 @@ checkFindWithoutPath = CommandCheck (Basename "find") f -- path. We assume that all the pre-path flags are single characters from a -- list of GNU and macOS flags. hasPath (first:rest) = - let flag = fromJust $ getLiteralStringExt (const $ return "___") first in + let flag = runIdentity $ getLiteralStringExt (const $ return "___") first in not ("-" `isPrefixOf` flag) || isLeadingFlag flag && hasPath rest hasPath [] = False isLeadingFlag flag = length flag <= 2 || all (`elem` leadingFlagChars) flag diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index b643525..d9ae1cf 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -30,6 +30,7 @@ import ShellCheck.Regex import Control.Monad import Control.Monad.RWS import Data.Char +import Data.Functor.Identity import Data.List import Data.Maybe import qualified Data.Map as Map @@ -488,7 +489,7 @@ checkBraceExpansionVars = ForShell [Bash] f T_DollarExpansion {} -> return "$" T_DollarArithmetic {} -> return "$" _ -> return "-" - toString t = fromJust $ getLiteralStringExt literalExt t + toString t = runIdentity $ getLiteralStringExt literalExt t isEvaled t = do cmd <- getClosestCommandM t return $ maybe False (`isUnqualifiedCommand` "eval") cmd From 4fd8de058b41caf77803c066e4009b4995875b3f Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sat, 8 Feb 2020 23:48:36 -0500 Subject: [PATCH 249/763] Remove more unnecessary uses of fromJust --- src/ShellCheck/Analytics.hs | 12 ++++++------ src/ShellCheck/Parser.hs | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 8a2902c..74c7398 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1996,18 +1996,18 @@ checkQuotesInLiterals params t = readF _ expr name = do assignment <- getQuotes name - return - (if isJust assignment - && not (isParamTo parents "eval" expr) + return $ case assignment of + Just j + | not (isParamTo parents "eval" expr) && not (isQuoteFree parents expr) && not (squashesQuotes expr) - then [ - makeComment WarningC (fromJust assignment) 2089 $ + -> [ + makeComment WarningC j 2089 $ "Quotes/backslashes will be treated literally. " ++ suggestion, makeComment WarningC (getId expr) 2090 "Quotes/backslashes in this variable will not be respected." ] - else []) + _ -> [] suggestion = if supportsArrays (shellType params) then "Use an array." diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 025fa98..0ef672f 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -2056,7 +2056,7 @@ readSimpleCommand = called "simple command" $ do firstArgument <- ignoreProblemsOf . optionMaybe . try . lookAhead $ readCmdWord suffix <- option [] $ getParser readCmdSuffix -- If `export` or other modifier commands are called with `builtin` we have to look at the first argument - (if isCommand ["builtin"] cmd && isJust firstArgument then fromJust firstArgument else cmd) [ + (if isCommand ["builtin"] cmd then fromMaybe cmd firstArgument else cmd) [ (["declare", "export", "local", "readonly", "typeset"], readModifierSuffix), (["time"], readTimeSuffix), (["let"], readLetSuffix), From f8648e546595a0e44d8f8e3c46a68ec774534bde Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 9 Feb 2020 21:26:42 -0500 Subject: [PATCH 250/763] Switch getLiteralStringExt to Identity where it can never be Nothing --- src/ShellCheck/Analytics.hs | 6 ++---- src/ShellCheck/Checks/Commands.hs | 9 ++++----- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 8a2902c..71743b4 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1066,9 +1066,7 @@ checkNumberComparisons params (TC_Binary id typ op lhs rhs) = do checkStrings = mapM_ stringError . take 1 . filter isNonNum - isNonNum t = fromMaybe False $ do - s <- getLiteralStringExt (const $ return "") t - return . not . all numChar $ s + isNonNum t = not . all numChar . runIdentity $ getLiteralStringExt (const $ return "") t numChar x = isDigit x || x `elem` "+-. " stringError t = err (getId t) 2170 $ @@ -2595,7 +2593,7 @@ checkTildeInPath _ (T_SimpleCommand _ vars _) = warn id 2147 "Literal tilde in PATH works poorly across programs." checkVar _ = return () - hasTilde t = fromMaybe False (liftM2 elem (return '~') (getLiteralStringExt (const $ return "") t)) + hasTilde t = '~' `elem` runIdentity (getLiteralStringExt (const $ return "") t) isQuoted T_DoubleQuoted {} = True isQuoted T_SingleQuoted {} = True isQuoted _ = False diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 869b389..a6b8d9e 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -249,13 +249,12 @@ prop_checkGrepRe23= verifyNot checkGrepRe "grep '.*' file" checkGrepRe = CommandCheck (Basename "grep") check where check cmd = f cmd (arguments cmd) -- --regex=*(extglob) doesn't work. Fixme? - skippable (Just s) = not ("--regex=" `isPrefixOf` s) && "-" `isPrefixOf` s - skippable _ = False + skippable s = not ("--regex=" `isPrefixOf` s) && "-" `isPrefixOf` s f _ [] = return () f cmd (x:r) = - let str = getLiteralStringExt (const $ return "_") x + let str = runIdentity $ getLiteralStringExt (const $ return "_") x in - if str `elem` [Just "--", Just "-e", Just "--regex"] + if str `elem` ["--", "-e", "--regex"] then checkRE cmd r -- Regex is *after* this else if skippable str @@ -366,7 +365,7 @@ checkFindExecWithSingleArgument = CommandCheck (Basename "find") (f . arguments) check (exec:arg:term:_) = do execS <- getLiteralString exec termS <- getLiteralString term - cmdS <- getLiteralStringExt (const $ return " ") arg + let cmdS = runIdentity $ getLiteralStringExt (const $ return " ") arg guard $ execS `elem` ["-exec", "-execdir"] && termS `elem` [";", "+"] guard $ cmdS `matches` commandRegex From 4d92a2e15c36c9a3c029b66144e41db42a83cfc3 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 9 Feb 2020 21:36:38 -0500 Subject: [PATCH 251/763] Add getLiteralStringDef and simplify with it --- src/ShellCheck/ASTLib.hs | 6 +++++- src/ShellCheck/Analytics.hs | 4 ++-- src/ShellCheck/AnalyzerLib.hs | 2 +- src/ShellCheck/Checks/Commands.hs | 8 ++++---- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 658ed64..17a475d 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -176,9 +176,13 @@ willConcatInAssignment token = getLiteralString :: Token -> Maybe String getLiteralString = getLiteralStringExt (const Nothing) +-- Definitely get a literal string, with a given default for all non-literals +getLiteralStringDef :: String -> Token -> String +getLiteralStringDef x = runIdentity . getLiteralStringExt (const $ return x) + -- Definitely get a literal string, skipping over all non-literals onlyLiteralString :: Token -> String -onlyLiteralString = runIdentity . getLiteralStringExt (const $ return "") +onlyLiteralString = getLiteralStringDef "" -- Maybe get a literal string, but only if it's an unquoted argument. getUnquotedLiteral (T_NormalWord _ list) = diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 71743b4..f390b53 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1066,7 +1066,7 @@ checkNumberComparisons params (TC_Binary id typ op lhs rhs) = do checkStrings = mapM_ stringError . take 1 . filter isNonNum - isNonNum t = not . all numChar . runIdentity $ getLiteralStringExt (const $ return "") t + isNonNum t = not . all numChar $ onlyLiteralString t numChar x = isDigit x || x `elem` "+-. " stringError t = err (getId t) 2170 $ @@ -2593,7 +2593,7 @@ checkTildeInPath _ (T_SimpleCommand _ vars _) = warn id 2147 "Literal tilde in PATH works poorly across programs." checkVar _ = return () - hasTilde t = '~' `elem` runIdentity (getLiteralStringExt (const $ return "") t) + hasTilde t = '~' `elem` onlyLiteralString t isQuoted T_DoubleQuoted {} = True isQuoted T_SingleQuoted {} = True isQuoted _ = False diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 0da80e3..7c046f3 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -806,7 +806,7 @@ isVariableName (x:r) = isVariableStartChar x && all isVariableChar r isVariableName _ = False getVariablesFromLiteralToken token = - getVariablesFromLiteral (runIdentity $ getLiteralStringExt (const $ return " ") token) + getVariablesFromLiteral (getLiteralStringDef " " token) -- Try to get referenced variables from a literal string like "$foo" -- Ignores tons of cases like arithmetic evaluation and array indices. diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index a6b8d9e..88b5967 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -252,7 +252,7 @@ checkGrepRe = CommandCheck (Basename "grep") check where skippable s = not ("--regex=" `isPrefixOf` s) && "-" `isPrefixOf` s f _ [] = return () f cmd (x:r) = - let str = runIdentity $ getLiteralStringExt (const $ return "_") x + let str = getLiteralStringDef "_" x in if str `elem` ["--", "-e", "--regex"] then checkRE cmd r -- Regex is *after* this @@ -365,7 +365,7 @@ checkFindExecWithSingleArgument = CommandCheck (Basename "find") (f . arguments) check (exec:arg:term:_) = do execS <- getLiteralString exec termS <- getLiteralString term - let cmdS = runIdentity $ getLiteralStringExt (const $ return " ") arg + let cmdS = getLiteralStringDef " " arg guard $ execS `elem` ["-exec", "-execdir"] && termS `elem` [";", "+"] guard $ cmdS `matches` commandRegex @@ -735,7 +735,7 @@ checkAliasesUsesArgs = CommandCheck (Exactly "alias") (f . arguments) re = mkRegex "\\$\\{?[0-9*@]" f = mapM_ checkArg checkArg arg = - let string = runIdentity $ getLiteralStringExt (const $ return "_") arg in + let string = getLiteralStringDef "_" arg in when ('=' `elem` string && string `matches` re) $ err (getId arg) 2142 "Aliases can't use positional parameters. Use a function." @@ -781,7 +781,7 @@ checkFindWithoutPath = CommandCheck (Basename "find") f -- path. We assume that all the pre-path flags are single characters from a -- list of GNU and macOS flags. hasPath (first:rest) = - let flag = runIdentity $ getLiteralStringExt (const $ return "___") first in + let flag = getLiteralStringDef "___" first in not ("-" `isPrefixOf` flag) || isLeadingFlag flag && hasPath rest hasPath [] = False isLeadingFlag flag = length flag <= 2 || all (`elem` leadingFlagChars) flag From 1e32139f6618953d60ce81860a3319062c92129b Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 9 Feb 2020 19:18:43 -0500 Subject: [PATCH 252/763] Replace mapMaybe and concatMap with list comprehensions --- src/ShellCheck/Analytics.hs | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 8a2902c..c58e97a 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -87,13 +87,8 @@ runList spec list = notes getEnableDirectives root = case root of - T_Annotation _ list _ -> mapMaybe getEnable list + T_Annotation _ list _ -> [s | EnableComment s <- list] _ -> [] - where - getEnable t = - case t of - EnableComment s -> return s - _ -> Nothing checkList l t = concatMap (\f -> f t) l @@ -3123,9 +3118,7 @@ checkUnmatchableCases params t = Just l -> " on line " <> show l <> "." _ -> "." - valids = concatMap f rest - f (x, Just y) = [(x,y)] - f _ = [] + valids = [(x,y) | (x, Just y) <- rest] checkDoms _ = return () From cb01cbf7eb3428e08a79bcc40ce4f54e4d1b3c7b Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 9 Feb 2020 19:33:36 -0500 Subject: [PATCH 253/763] Use mapM instead of implementing a slower version of it --- src/ShellCheck/Analytics.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index c58e97a..e91810b 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -431,7 +431,7 @@ checkWrongArithmeticAssignment params (T_SimpleCommand id (T_Assignment _ _ _ _ insertRef _ = Prelude.id getNormalString (T_NormalWord _ words) = do - parts <- foldl (liftM2 (\x y -> x ++ [y])) (Just []) $ map getLiterals words + parts <- mapM getLiterals words return $ concat parts getNormalString _ = Nothing From cc424bac11864e5c445b18490e0507457b86b5a5 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 9 Feb 2020 19:40:57 -0500 Subject: [PATCH 254/763] Use find instead of take 1 and filter --- src/ShellCheck/Analytics.hs | 6 +++--- src/ShellCheck/Checks/Commands.hs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index e91810b..fa640b1 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -790,7 +790,7 @@ prop_checkUnquotedDollarAt8 = verifyNot checkUnquotedDollarAt "echo \"${args[@]: prop_checkUnquotedDollarAt9 = verifyNot checkUnquotedDollarAt "echo ${args[@]:+\"${args[@]}\"}" prop_checkUnquotedDollarAt10 = verifyNot checkUnquotedDollarAt "echo ${@+\"$@\"}" checkUnquotedDollarAt p word@(T_NormalWord _ parts) | not $ isStrictlyQuoteFree (parentMap p) word = - forM_ (take 1 $ filter isArrayExpansion parts) $ \x -> + forM_ (find isArrayExpansion parts) $ \x -> unless (isQuotedAlternativeReference x) $ err (getId x) 2068 "Double quote array expansions to avoid re-splitting elements." @@ -807,7 +807,7 @@ checkConcatenatedDollarAt p word@T_NormalWord {} mapM_ for array where parts = getWordParts word - array = take 1 $ filter isArrayExpansion parts + array = find isArrayExpansion parts for t = err (getId t) 2145 "Argument mixes string and array. Use * or separate argument." checkConcatenatedDollarAt _ _ = return () @@ -1059,7 +1059,7 @@ checkNumberComparisons params (TC_Binary id typ op lhs rhs) = do "Either use integers only, or use bc or awk to compare." checkStrings = - mapM_ stringError . take 1 . filter isNonNum + mapM_ stringError . find isNonNum isNonNum t = fromMaybe False $ do s <- getLiteralStringExt (const $ return "") t diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 299a335..881dd9d 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -748,7 +748,7 @@ checkAliasesExpandEarly = CommandCheck (Exactly "alias") (f . arguments) where f = mapM_ checkArg checkArg arg | '=' `elem` concat (oversimplify arg) = - forM_ (take 1 $ filter (not . isLiteral) $ getWordParts arg) $ + forM_ (find (not . isLiteral) $ getWordParts arg) $ \x -> warn (getId x) 2139 "This expands when defined, not when used. Consider escaping." checkArg _ = return () From ffbbfcfe25a9d03fde4ce6f65e0fb0852d3bc6e6 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 9 Feb 2020 19:53:18 -0500 Subject: [PATCH 255/763] Use mapM_ and sequence_ instead of reimplementing them --- src/ShellCheck/ASTLib.hs | 2 +- src/ShellCheck/Analytics.hs | 12 +++++------- src/ShellCheck/Checks/Commands.hs | 6 +++--- src/ShellCheck/Parser.hs | 2 +- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 03a2f9a..2e08f4d 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -382,7 +382,7 @@ getAssociativeArrays t = nub . execWriter $ doAnalysis f t where f :: Token -> Writer [String] () - f t@T_SimpleCommand {} = fromMaybe (return ()) $ do + f t@T_SimpleCommand {} = sequence_ $ do name <- getCommandName t let assocNames = ["declare","local","typeset"] guard $ elem name assocNames diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index fa640b1..b4b3096 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -404,7 +404,7 @@ prop_checkArithmeticOpCommand1 = verify checkArithmeticOpCommand "i=i + 1" prop_checkArithmeticOpCommand2 = verify checkArithmeticOpCommand "foo=bar * 2" prop_checkArithmeticOpCommand3 = verifyNot checkArithmeticOpCommand "foo + opts" checkArithmeticOpCommand _ (T_SimpleCommand id [T_Assignment {}] (firstWord:_)) = - maybe (return ()) check $ getGlobOrLiteralString firstWord + mapM_ check $ getGlobOrLiteralString firstWord where check op = when (op `elem` ["+", "-", "*", "/"]) $ @@ -415,7 +415,7 @@ checkArithmeticOpCommand _ _ = return () prop_checkWrongArit = verify checkWrongArithmeticAssignment "i=i+1" prop_checkWrongArit2 = verify checkWrongArithmeticAssignment "n=2; i=n*2" checkWrongArithmeticAssignment params (T_SimpleCommand id (T_Assignment _ _ _ _ val:[]) []) = - fromMaybe (return ()) $ do + sequence_ $ do str <- getNormalString val match <- matchRegex regex str var <- match !!! 0 @@ -2524,7 +2524,7 @@ checkUnpassedInFunctions params root = referenceList :: [(String, Bool, Token)] referenceList = execWriter $ - doAnalysis (fromMaybe (return ()) . checkCommand) root + doAnalysis (sequence_ . checkCommand) root checkCommand :: Token -> Maybe (Writer [(String, Bool, Token)] ()) checkCommand t@(T_SimpleCommand _ _ (cmd:args)) = do str <- getLiteralString cmd @@ -2648,9 +2648,7 @@ prop_checkSuspiciousIFS1 = verify checkSuspiciousIFS "IFS=\"\\n\"" prop_checkSuspiciousIFS2 = verifyNot checkSuspiciousIFS "IFS=$'\\t'" prop_checkSuspiciousIFS3 = verify checkSuspiciousIFS "IFS=' \\t\\n'" checkSuspiciousIFS params (T_Assignment _ _ "IFS" [] value) = - potentially $ do - str <- getLiteralString value - return $ check str + mapM_ check $ getLiteralString value where hasDollarSingle = shellType params == Bash || shellType params == Ksh n = if hasDollarSingle then "$'\\n'" else "''" @@ -3465,7 +3463,7 @@ prop_checkTranslatedStringVariable3 = verifyNot checkTranslatedStringVariable "$ prop_checkTranslatedStringVariable4 = verifyNot checkTranslatedStringVariable "var=val; $\"$var\"" prop_checkTranslatedStringVariable5 = verifyNot checkTranslatedStringVariable "foo=var; bar=val2; $\"foo bar\"" checkTranslatedStringVariable params (T_DollarDoubleQuoted id [T_Literal _ s]) = - fromMaybe (return ()) $ do + sequence_ $ do guard $ all isVariableChar s Map.lookup s assignments return $ diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 881dd9d..4842341 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -122,7 +122,7 @@ buildCommandMap = foldl' addCheck Map.empty checkCommand :: Map.Map CommandName (Token -> Analysis) -> Token -> Analysis -checkCommand map t@(T_SimpleCommand id cmdPrefix (cmd:rest)) = fromMaybe (return ()) $ do +checkCommand map t@(T_SimpleCommand id cmdPrefix (cmd:rest)) = sequence_ $ do name <- getLiteralString cmd return $ if '/' `elem` name @@ -575,7 +575,7 @@ checkPrintfVar = CommandCheck (Exactly "printf") (f . arguments) where f _ = return () check format more = do - fromMaybe (return ()) $ do + sequence_ $ do string <- getLiteralString format let formats = getPrintfFormats string let formatCount = length formats @@ -945,7 +945,7 @@ checkCatastrophicRm = CommandCheck (Basename "rm") $ \t -> Nothing -> checkWord' token - checkWord' token = fromMaybe (return ()) $ do + checkWord' token = sequence_ $ do filename <- getPotentialPath token let path = fixPath filename return . when (path `elem` importantPaths) $ diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 025fa98..30529e0 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -586,7 +586,7 @@ readConditionContents single = return $ TC_Nullary id typ x ) - checkTrailingOp x = fromMaybe (return ()) $ do + checkTrailingOp x = sequence_ $ do (T_Literal id str) <- getTrailingUnquotedLiteral x trailingOp <- find (`isSuffixOf` str) binaryTestOps return $ parseProblemAtId id ErrorC 1108 $ From 4bfe6496d94786ebd8c234638aee89622cc4868a Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 9 Feb 2020 20:09:25 -0500 Subject: [PATCH 256/763] Simplify check and checkTranslatedStringVariable Avoid the "potentially" and "Maybe" business, and just use regular guards. --- src/ShellCheck/Analytics.hs | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index b4b3096..91345ed 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2933,16 +2933,15 @@ checkTrailingBracket _ token = T_SimpleCommand _ _ tokens@(_:_) -> check (last tokens) token _ -> return () where - check t command = - case t of - T_NormalWord id [T_Literal _ str] -> potentially $ do - guard $ str `elem` [ "]]", "]" ] - let opposite = invert str - parameters = oversimplify command - guard $ opposite `notElem` parameters - return $ warn id 2171 $ - "Found trailing " ++ str ++ " outside test. Add missing " ++ opposite ++ " or quote if intentional." - _ -> return () + check (T_NormalWord id [T_Literal _ str]) command + | str `elem` [ "]]", "]" ] + && opposite `notElem` parameters + = warn id 2171 $ + "Found trailing " ++ str ++ " outside test. Add missing " ++ opposite ++ " or quote if intentional." + where + opposite = invert str + parameters = oversimplify command + check _ _ = return () invert s = case s of "]]" -> "[[" @@ -3462,12 +3461,10 @@ prop_checkTranslatedStringVariable2 = verifyNot checkTranslatedStringVariable "$ prop_checkTranslatedStringVariable3 = verifyNot checkTranslatedStringVariable "$\"..\"" prop_checkTranslatedStringVariable4 = verifyNot checkTranslatedStringVariable "var=val; $\"$var\"" prop_checkTranslatedStringVariable5 = verifyNot checkTranslatedStringVariable "foo=var; bar=val2; $\"foo bar\"" -checkTranslatedStringVariable params (T_DollarDoubleQuoted id [T_Literal _ s]) = - sequence_ $ do - guard $ all isVariableChar s - Map.lookup s assignments - return $ - warnWithFix id 2256 "This translated string is the name of a variable. Flip leading $ and \" if this should be a quoted substitution." (fix id) +checkTranslatedStringVariable params (T_DollarDoubleQuoted id [T_Literal _ s]) + | all isVariableChar s + && Map.member s assignments + = warnWithFix id 2256 "This translated string is the name of a variable. Flip leading $ and \" if this should be a quoted substitution." (fix id) where assignments = foldl (flip ($)) Map.empty (map insertAssignment $ variableFlow params) insertAssignment (Assignment (_, token, name, _)) | isVariableName name = From 7e6a556ef155bb50f554235b93097928145dc74e Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 9 Feb 2020 20:10:09 -0500 Subject: [PATCH 257/763] Get rid of potentially This already exists as sequence_. --- src/ShellCheck/Analytics.hs | 38 +++++++++++++-------------- src/ShellCheck/AnalyzerLib.hs | 9 ------- src/ShellCheck/Checks/Commands.hs | 18 ++++++------- src/ShellCheck/Checks/ShellSupport.hs | 8 +++--- 4 files changed, 32 insertions(+), 41 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 91345ed..4100b2d 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1209,7 +1209,7 @@ prop_checkLiteralBreakingTest6 = verify checkLiteralBreakingTest "[ -z $(true)z prop_checkLiteralBreakingTest7 = verifyNot checkLiteralBreakingTest "[ -z $(true) ]" prop_checkLiteralBreakingTest8 = verifyNot checkLiteralBreakingTest "[ $(true)$(true) ]" prop_checkLiteralBreakingTest10 = verify checkLiteralBreakingTest "[ -z foo ]" -checkLiteralBreakingTest _ t = potentially $ +checkLiteralBreakingTest _ t = sequence_ $ case t of (TC_Nullary _ _ w@(T_NormalWord _ l)) -> do guard . not $ isConstant w -- Covered by SC2078 @@ -1257,7 +1257,7 @@ checkConstantNullary _ _ = return () prop_checkForDecimals1 = verify checkForDecimals "((3.14*c))" prop_checkForDecimals2 = verify checkForDecimals "foo[1.2]=bar" prop_checkForDecimals3 = verifyNot checkForDecimals "declare -A foo; foo[1.2]=bar" -checkForDecimals params t@(TA_Expansion id _) = potentially $ do +checkForDecimals params t@(TA_Expansion id _) = sequence_ $ do guard $ not (hasFloatingPoint params) str <- getLiteralString t first <- str !!! 0 @@ -1310,7 +1310,7 @@ checkArithmeticDeref _ _ = return () prop_checkArithmeticBadOctal1 = verify checkArithmeticBadOctal "(( 0192 ))" prop_checkArithmeticBadOctal2 = verifyNot checkArithmeticBadOctal "(( 0x192 ))" prop_checkArithmeticBadOctal3 = verifyNot checkArithmeticBadOctal "(( 1 ^ 0777 ))" -checkArithmeticBadOctal _ t@(TA_Expansion id _) = potentially $ do +checkArithmeticBadOctal _ t@(TA_Expansion id _) = sequence_ $ do str <- getLiteralString t guard $ str `matches` octalRE return $ err id 2080 "Numbers with leading 0 are considered octal." @@ -1392,7 +1392,7 @@ checkOrNeq _ (TA_Binary id "||" (TA_Binary _ "!=" word1 _) (TA_Binary _ "!=" wor warn id 2056 "You probably wanted && here, otherwise it's always true." -- For command level "or": [ x != y ] || [ x != z ] -checkOrNeq _ (T_OrIf id lhs rhs) = potentially $ do +checkOrNeq _ (T_OrIf id lhs rhs) = sequence_ $ do (lhs1, op1, rhs1) <- getExpr lhs (lhs2, op2, rhs2) <- getExpr rhs guard $ op1 == op2 && op1 `elem` ["-ne", "!="] @@ -1407,7 +1407,7 @@ checkOrNeq _ (T_OrIf id lhs rhs) = potentially $ do T_Redirecting _ _ c -> getExpr c T_Condition _ _ c -> getExpr c TC_Binary _ _ op lhs rhs -> return (lhs, op, rhs) - _ -> fail "" + _ -> Nothing checkOrNeq _ _ = return () @@ -2068,7 +2068,7 @@ checkFunctionsUsedExternally params t = in when ('=' `elem` string) $ modify ((takeWhile (/= '=') string, getId arg):) - checkArg cmd (_, arg) = potentially $ do + checkArg cmd (_, arg) = sequence_ $ do literalArg <- getUnquotedLiteral arg -- only consider unquoted literals definitionId <- Map.lookup literalArg functions return $ do @@ -2312,7 +2312,7 @@ checkWhileReadPitfalls _ (T_WhileExpression id [command] contents) (T_IfExpression _ thens elses) -> mapM_ checkMuncher . concat $ map fst thens ++ map snd thens ++ [elses] - _ -> potentially $ do + _ -> sequence_ $ do name <- getCommandBasename cmd guard $ name `elem` munchers @@ -2410,7 +2410,7 @@ checkCdAndBack params t = else findCdPair (b:rest) _ -> Nothing - doList list = potentially $ do + doList list = sequence_ $ do cd <- findCdPair $ mapMaybe getCandidate list return $ info cd 2103 "Use a ( subshell ) to avoid having to cd back." @@ -2673,7 +2673,7 @@ prop_checkGrepQ4= verifyNot checkShouldUseGrepQ "[ -z $(grep bar | cmd) ]" prop_checkGrepQ5= verifyNot checkShouldUseGrepQ "rm $(ls | grep file)" prop_checkGrepQ6= verifyNot checkShouldUseGrepQ "[[ -n $(pgrep foo) ]]" checkShouldUseGrepQ params t = - potentially $ case t of + sequence_ $ case t of TC_Nullary id _ token -> check id True token TC_Unary id _ "-n" token -> check id True token TC_Unary id _ "-z" token -> check id False token @@ -2807,7 +2807,7 @@ prop_checkMaskedReturns2 = verify checkMaskedReturns "declare a=$(false)" prop_checkMaskedReturns3 = verify checkMaskedReturns "declare a=\"`false`\"" prop_checkMaskedReturns4 = verifyNot checkMaskedReturns "declare a; a=$(false)" prop_checkMaskedReturns5 = verifyNot checkMaskedReturns "f() { local -r a=$(false); }" -checkMaskedReturns _ t@(T_SimpleCommand id _ (cmd:rest)) = potentially $ do +checkMaskedReturns _ t@(T_SimpleCommand id _ (cmd:rest)) = sequence_ $ do name <- getCommandName t guard $ name `elem` ["declare", "export"] || name == "local" && "r" `notElem` map snd (getAllFlags t) @@ -2900,7 +2900,7 @@ prop_checkLoopVariableReassignment1 = verify checkLoopVariableReassignment "for prop_checkLoopVariableReassignment2 = verify checkLoopVariableReassignment "for i in *; do for((i=0; i<3; i++)); do true; done; done" prop_checkLoopVariableReassignment3 = verifyNot checkLoopVariableReassignment "for i in *; do for j in *.bar; do true; done; done" checkLoopVariableReassignment params token = - potentially $ case token of + sequence_ $ case token of T_ForIn {} -> check T_ForArithmetic {} -> check _ -> Nothing @@ -2988,12 +2988,12 @@ prop_checkRedirectedNowhere7 = verifyNot checkRedirectedNowhere "var=$(< file)" prop_checkRedirectedNowhere8 = verifyNot checkRedirectedNowhere "var=`< file`" checkRedirectedNowhere params token = case token of - T_Pipeline _ _ [single] -> potentially $ do + T_Pipeline _ _ [single] -> sequence_ $ do redir <- getDanglingRedirect single guard . not $ isInExpansion token return $ warn (getId redir) 2188 "This redirection doesn't have a command. Move to its command (or use 'true' as no-op)." - T_Pipeline _ _ list -> forM_ list $ \x -> potentially $ do + T_Pipeline _ _ list -> forM_ list $ \x -> sequence_ $ do redir <- getDanglingRedirect x return $ err (getId redir) 2189 "You can't have | between this redirection and the command it should apply to." @@ -3080,7 +3080,7 @@ checkUnmatchableCases params t = if isConstant word then warn (getId word) 2194 "This word is constant. Did you forget the $ on a variable?" - else potentially $ do + else sequence_ $ do pg <- wordToPseudoGlob word return $ mapM_ (check pg) allpatterns @@ -3095,7 +3095,7 @@ checkUnmatchableCases params t = fst3 (x,_,_) = x snd3 (_,x,_) = x tp = tokenPositions params - check target candidate = potentially $ do + check target candidate = sequence_ $ do candidateGlob <- wordToPseudoGlob candidate guard . not $ pseudoGlobsCanOverlap target candidateGlob return $ warn (getId candidate) 2195 @@ -3189,7 +3189,7 @@ prop_checkRedirectionToNumber2 = verify checkRedirectionToNumber "foo 1>2" prop_checkRedirectionToNumber3 = verifyNot checkRedirectionToNumber "echo foo > '2'" prop_checkRedirectionToNumber4 = verifyNot checkRedirectionToNumber "foo 1>&2" checkRedirectionToNumber _ t = case t of - T_IoFile id _ word -> potentially $ do + T_IoFile id _ word -> sequence_ $ do file <- getUnquotedLiteral word guard $ all isDigit file return $ warn id 2210 "This is a file redirection. Was it supposed to be a comparison or fd operation?" @@ -3238,7 +3238,7 @@ checkPipeToNowhere _ t = T_Redirecting _ redirects cmd -> when (any redirectsStdin redirects) $ checkRedir cmd _ -> return () where - checkPipe redir = potentially $ do + checkPipe redir = sequence_ $ do cmd <- getCommand redir name <- getCommandBasename cmd guard $ name `elem` nonReadingCommands @@ -3251,7 +3251,7 @@ checkPipeToNowhere _ t = return $ warn (getId cmd) 2216 $ "Piping to '" ++ name ++ "', a command that doesn't read stdin. " ++ suggestion - checkRedir cmd = potentially $ do + checkRedir cmd = sequence_ $ do name <- getCommandBasename cmd guard $ name `elem` nonReadingCommands guard . not $ hasAdditionalConsumers cmd @@ -3299,7 +3299,7 @@ checkUseBeforeDefinition _ t = mapM_ (checkUsage m) $ concatMap recursiveSequences cmds _ -> return () - checkUsage map cmd = potentially $ do + checkUsage map cmd = sequence_ $ do name <- getCommandName cmd def <- Map.lookup name map return $ diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index e4640e7..69eaf63 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -870,15 +870,6 @@ getBracedModifier s = fromMaybe "" . listToMaybe $ do -- Useful generic functions. --- Run an action in a Maybe (or do nothing). --- Example: --- potentially $ do --- s <- getLiteralString cmd --- guard $ s `elem` ["--recursive", "-r"] --- return $ warn .. "Something something recursive" -potentially :: Monad m => Maybe (m ()) -> m () -potentially = fromMaybe (return ()) - -- Get element 0 or a default. Like `head` but safe. headOrDefault _ (a:_) = a headOrDefault def _ = def diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 4842341..97de0f6 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -270,7 +270,7 @@ checkGrepRe = CommandCheck (Basename "grep") check where let string = concat $ oversimplify re if isConfusedGlobRegex string then warn (getId re) 2063 "Grep uses regex, but this looks like a glob." - else potentially $ do + else sequence_ $ do char <- getSuspiciousRegexWildcard string return $ info (getId re) 2022 $ "Note that unlike globs, " ++ [char] ++ "* here matches '" ++ [char, char, char] ++ "' but not '" ++ wordStartingWith char ++ "'." @@ -461,7 +461,7 @@ prop_checkMkdirDashPM20 = verifyNot checkMkdirDashPM "mkdir -p -m 0755 .././bin" prop_checkMkdirDashPM21 = verifyNot checkMkdirDashPM "mkdir -p -m 0755 ../../bin" checkMkdirDashPM = CommandCheck (Basename "mkdir") check where - check t = potentially $ do + check t = sequence_ $ do let flags = getAllFlags t dashP <- find ((\f -> f == "p" || f == "parents") . snd) flags dashM <- find ((\f -> f == "m" || f == "mode") . snd) flags @@ -487,7 +487,7 @@ checkNonportableSignals = CommandCheck (Exactly "trap") (f . arguments) first:rest -> unless (isFlag first) $ mapM_ check rest _ -> return () - check param = potentially $ do + check param = sequence_ $ do str <- getLiteralString param let id = getId param return $ sequence_ $ mapMaybe (\f -> f id str) [ @@ -687,7 +687,7 @@ prop_checkExportedExpansions3 = verifyNot checkExportedExpansions "export foo" prop_checkExportedExpansions4 = verifyNot checkExportedExpansions "export ${foo?}" checkExportedExpansions = CommandCheck (Exactly "export") (mapM_ check . arguments) where - check t = potentially $ do + check t = sequence_ $ do var <- getSingleUnmodifiedVariable t let name = bracedString var return . warn (getId t) 2163 $ @@ -709,7 +709,7 @@ checkReadExpansions = CommandCheck (Exactly "read") check return [y | (x,y) <- opts, null x || x == "a"] check cmd = mapM_ warning $ getVars cmd - warning t = potentially $ do + warning t = sequence_ $ do var <- getSingleUnmodifiedVariable t let name = bracedString var guard $ isVariableName name -- e.g. not $1 @@ -859,7 +859,7 @@ checkWhileGetoptsCase = CommandCheck (Exactly "getopts") f f :: Token -> Analysis f t@(T_SimpleCommand _ _ (cmd:arg1:_)) = do path <- getPathM t - potentially $ do + sequence_ $ do options <- getLiteralString arg1 (T_WhileExpression _ _ body) <- findFirst whileLoop path caseCmd <- mapMaybe findCase body !!! 0 @@ -886,7 +886,7 @@ checkWhileGetoptsCase = CommandCheck (Exactly "getopts") f warnUnhandled optId caseId str = warn caseId 2213 $ "getopts specified -" ++ str ++ ", but it's not handled by this 'case'." - warnRedundant (key, expr) = potentially $ do + warnRedundant (key, expr) = sequence_ $ do str <- key guard $ str `notElem` ["*", ":", "?"] return $ warn (getId expr) 2214 "This case is not specified by getopts." @@ -1081,7 +1081,7 @@ prop_checkSudoArgs6 = verifyNot checkSudoArgs "sudo -n -u export ls" prop_checkSudoArgs7 = verifyNot checkSudoArgs "sudo docker export foo" checkSudoArgs = CommandCheck (Basename "sudo") f where - f t = potentially $ do + f t = sequence_ $ do opts <- parseOpts t let nonFlags = [x | ("",x) <- opts] commandArg <- nonFlags !!! 0 @@ -1109,7 +1109,7 @@ prop_checkChmodDashr3 = verifyNot checkChmodDashr "chmod a-r dir" checkChmodDashr = CommandCheck (Basename "chmod") f where f t = mapM_ check $ arguments t - check t = potentially $ do + check t = sequence_ $ do flag <- getLiteralString t guard $ flag == "-r" return $ warn (getId t) 2253 "Use -R to recurse, or explicitly a-r to remove read permissions." diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index b643525..4e0655d 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -73,7 +73,7 @@ prop_checkForDecimals2 = verify checkForDecimals "foo[1.2]=bar" prop_checkForDecimals3 = verifyNot checkForDecimals "declare -A foo; foo[1.2]=bar" checkForDecimals = ForShell [Sh, Dash, Bash] f where - f t@(TA_Expansion id _) = potentially $ do + f t@(TA_Expansion id _) = sequence_ $ do str <- getLiteralString t first <- str !!! 0 guard $ isDigit first && '.' `elem` str @@ -337,7 +337,7 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do in do when (name `elem` unsupportedCommands) $ warnMsg id $ "'" ++ name ++ "' is" - potentially $ do + sequence_ $ do allowed' <- Map.lookup name allowedFlags allowed <- allowed' (word, flag) <- find @@ -347,7 +347,7 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do when (name == "source") $ warnMsg id "'source' in place of '.' is" when (name == "trap") $ let - check token = potentially $ do + check token = sequence_ $ do str <- getLiteralString token let upper = map toUpper str return $ do @@ -362,7 +362,7 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do in mapM_ check (drop 1 rest) - when (name == "printf") $ potentially $ do + when (name == "printf") $ sequence_ $ do format <- rest !!! 0 -- flags are covered by allowedFlags let literal = onlyLiteralString format guard $ "%q" `isInfixOf` literal From 0ca50159ec24f2aa6495317927a943bfd69272c2 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 9 Feb 2020 20:12:57 -0500 Subject: [PATCH 258/763] Use head instead of reimplementing it Normally I wouldn't use head, but this code is partial anyway. --- src/ShellCheck/Analytics.hs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 4100b2d..cfe9301 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -565,10 +565,8 @@ checkShebang params (T_Script _ (T_Literal id sb) _) = execWriter $ do unless (null sb) $ do unless ("/" `isPrefixOf` sb) $ err id 2239 "Ensure the shebang uses an absolute path to the interpreter." - case words sb of - first:_ -> - when ("/" `isSuffixOf` first) $ - err id 2246 "This shebang specifies a directory. Ensure the interpreter is a file." + when ("/" `isSuffixOf` head (words sb)) $ + err id 2246 "This shebang specifies a directory. Ensure the interpreter is a file." prop_checkForInQuoted = verify checkForInQuoted "for f in \"$(ls)\"; do echo foo; done" From 0e00249eaed3a13e78bdcd9f146661963959f685 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 9 Feb 2020 20:22:06 -0500 Subject: [PATCH 259/763] Use void instead of do and return () --- src/ShellCheck/Analytics.hs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index cfe9301..6bd1e5d 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -500,11 +500,10 @@ checkPipePitfalls _ (T_Pipeline id _ commands) = do for' ["ls", "xargs"] $ \x -> warn x 2011 "Use 'find .. -print0 | xargs -0 ..' or 'find .. -exec .. +' to allow non-alphanumeric filenames." ] - unless didLs $ do + unless didLs $ void $ for ["ls", "?"] $ \(ls:_) -> unless (hasShortParameter 'N' (oversimplify ls)) $ info (getId ls) 2012 "Use find instead of ls to better handle non-alphanumeric filenames." - return () where for l f = let indices = indexOfSublists l (map (headOrDefault "" . oversimplify) commands) From 057cc714b3afc3530ab07978bd24320e84ab1137 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 9 Feb 2020 20:36:14 -0500 Subject: [PATCH 260/763] Simplify matchToken --- src/ShellCheck/Analytics.hs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 6bd1e5d..854dac7 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1220,10 +1220,7 @@ checkLiteralBreakingTest _ t = sequence_ $ where hasEquals = matchToken ('=' `elem`) isNonEmpty = matchToken (not . null) - matchToken m t = isJust $ do - str <- getLiteralString t - guard $ m str - return () + matchToken m t = maybe False m (getLiteralString t) comparisonWarning list = do token <- find hasEquals list From a6efd02807ad7ae6c8047d4f42fb46150c8bd018 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 9 Feb 2020 20:45:05 -0500 Subject: [PATCH 261/763] Simplify <> for SpaceStatus --- src/ShellCheck/Analytics.hs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 854dac7..9581b7c 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1804,13 +1804,11 @@ prop_checkSpacefulness40= verifyNotTree checkSpacefulness "a=$((x+1)); echo $a" data SpaceStatus = SpaceSome | SpaceNone | SpaceEmpty deriving (Eq) instance Semigroup SpaceStatus where - (<>) x y = - case (x,y) of - (SpaceNone, SpaceNone) -> SpaceNone - (SpaceSome, _) -> SpaceSome - (_, SpaceSome) -> SpaceSome - (SpaceEmpty, x) -> x - (x, SpaceEmpty) -> x + SpaceNone <> SpaceNone = SpaceNone + SpaceSome <> _ = SpaceSome + _ <> SpaceSome = SpaceSome + SpaceEmpty <> x = x + x <> SpaceEmpty = x instance Monoid SpaceStatus where mempty = SpaceEmpty mappend = (<>) From c290eace543f35ce15b89432f97ef1b0e4b240d3 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 9 Feb 2020 20:51:41 -0500 Subject: [PATCH 262/763] Inline an uncurry --- src/ShellCheck/Analytics.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 9581b7c..b02ad0f 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2223,14 +2223,14 @@ checkUnassignedReferences' includeGlobals params t = warnings match <- getBestMatch var return $ " (did you mean '" ++ match ++ "'?)" - warningFor var place = do + warningFor (var, place) = do guard $ isVariableName var guard . not $ isInArray var place || isGuarded place (if includeGlobals || isLocal var then warningForLocals else warningForGlobals) var place - warnings = execWriter . sequence $ mapMaybe (uncurry warningFor) unassigned + warnings = execWriter . sequence $ mapMaybe warningFor unassigned -- Due to parsing, foo=( [bar]=baz ) parses 'bar' as a reference even for assoc arrays. -- Similarly, ${foo[bar baz]} may not be referencing bar/baz. Just skip these. From 172aa7c4fc1014cb2418471e2eb73ac1f43fd3ff Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 9 Feb 2020 20:55:49 -0500 Subject: [PATCH 263/763] Avoid unnecessary use of when and unless --- src/ShellCheck/Analytics.hs | 124 ++++++++++++++++++------------------ 1 file changed, 61 insertions(+), 63 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index b02ad0f..5f19352 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -257,12 +257,12 @@ verifyTree f s = producesComments f s == Just True verifyNotTree :: (Parameters -> Token -> [TokenComment]) -> String -> Bool verifyNotTree f s = producesComments f s == Just False -checkCommand str f t@(T_SimpleCommand id _ (cmd:rest)) = - when (t `isCommand` str) $ f cmd rest +checkCommand str f t@(T_SimpleCommand id _ (cmd:rest)) + | t `isCommand` str = f cmd rest checkCommand _ _ _ = return () -checkUnqualifiedCommand str f t@(T_SimpleCommand id _ (cmd:rest)) = - when (t `isUnqualifiedCommand` str) $ f cmd rest +checkUnqualifiedCommand str f t@(T_SimpleCommand id _ (cmd:rest)) + | t `isUnqualifiedCommand` str = f cmd rest checkUnqualifiedCommand _ _ _ = return () @@ -450,7 +450,7 @@ prop_checkUuoc6 = verifyNot checkUuoc "cat -n | grep bar" checkUuoc _ (T_Pipeline _ _ (T_Redirecting _ _ cmd:_:_)) = checkCommand "cat" (const f) cmd where - f [word] = unless (mayBecomeMultipleArgs word || isOption word) $ + f [word] | not (mayBecomeMultipleArgs word || isOption word) = style (getId word) 2002 "Useless cat. Consider 'cmd < file | ..' or 'cmd file | ..' instead." f _ = return () isOption word = "-" `isPrefixOf` onlyLiteralString word @@ -577,16 +577,15 @@ prop_checkForInQuoted4 = verify checkForInQuoted "for f in 1,2,3; do true; done" prop_checkForInQuoted4a = verifyNot checkForInQuoted "for f in foo{1,2,3}; do true; done" prop_checkForInQuoted5 = verify checkForInQuoted "for f in ls; do true; done" prop_checkForInQuoted6 = verifyNot checkForInQuoted "for f in \"${!arr}\"; do true; done" -checkForInQuoted _ (T_ForIn _ f [T_NormalWord _ [word@(T_DoubleQuoted id list)]] _) = - when (any (\x -> willSplit x && not (mayBecomeMultipleArgs x)) list - || (fmap wouldHaveBeenGlob (getLiteralString word) == Just True)) $ +checkForInQuoted _ (T_ForIn _ f [T_NormalWord _ [word@(T_DoubleQuoted id list)]] _) + | any (\x -> willSplit x && not (mayBecomeMultipleArgs x)) list + || (fmap wouldHaveBeenGlob (getLiteralString word) == Just True) = err id 2066 "Since you double quoted this, it will not word split, and the loop will only run once." checkForInQuoted _ (T_ForIn _ f [T_NormalWord _ [T_SingleQuoted id _]] _) = warn id 2041 "This is a literal string. To run as a command, use $(..) instead of '..' . " checkForInQuoted _ (T_ForIn _ f [T_NormalWord _ [T_Literal id s]] _) = - if ',' `elem` s - then unless ('{' `elem` s) $ - warn id 2042 "Use spaces, not commas, to separate loop elements." + if ',' `elem` s && '{' `notElem` s + then warn id 2042 "Use spaces, not commas, to separate loop elements." else warn id 2043 "This loop will only ever run once for a constant value. Did you perhaps mean to loop over dir/*, $var or $(cmd)?" checkForInQuoted _ _ = return () @@ -705,13 +704,13 @@ checkRedirectToSame params s@(T_Pipeline _ _ list) = where note x = makeComment InfoC x 2094 "Make sure not to read and write the same file in the same pipeline." - checkOccurrences t@(T_NormalWord exceptId x) u@(T_NormalWord newId y) = - when (exceptId /= newId + checkOccurrences t@(T_NormalWord exceptId x) u@(T_NormalWord newId y) | + exceptId /= newId && x == y && not (isOutput t && isOutput u) && not (special t) && not (any isHarmlessCommand [t,u]) - && not (any containsAssignment [u])) $ do + && not (any containsAssignment [u]) = do addComment $ note newId addComment $ note exceptId checkOccurrences _ _ = return () @@ -769,9 +768,9 @@ prop_checkDollarStar = verify checkDollarStar "for f in $*; do ..; done" prop_checkDollarStar2 = verifyNot checkDollarStar "a=$*" prop_checkDollarStar3 = verifyNot checkDollarStar "[[ $* = 'a b' ]]" checkDollarStar p t@(T_NormalWord _ [b@(T_DollarBraced id _ _)]) - | bracedString b == "*" = - unless (isStrictlyQuoteFree (parentMap p) t) $ - warn id 2048 "Use \"$@\" (with quotes) to prevent whitespace problems." + | bracedString b == "*" && + not (isStrictlyQuoteFree (parentMap p) t) = + warn id 2048 "Use \"$@\" (with quotes) to prevent whitespace problems." checkDollarStar _ _ = return () @@ -799,9 +798,9 @@ prop_checkConcatenatedDollarAt3 = verify checkConcatenatedDollarAt "echo $a$@" prop_checkConcatenatedDollarAt4 = verifyNot checkConcatenatedDollarAt "echo $@" prop_checkConcatenatedDollarAt5 = verifyNot checkConcatenatedDollarAt "echo \"${arr[@]}\"" checkConcatenatedDollarAt p word@T_NormalWord {} - | not $ isQuoteFree (parentMap p) word = - unless (null $ drop 1 parts) $ - mapM_ for array + | not $ isQuoteFree (parentMap p) word + || null (drop 1 parts) = + mapM_ for array where parts = getWordParts word array = find isArrayExpansion parts @@ -1099,8 +1098,8 @@ checkNumberComparisons params (TC_Binary id typ op lhs rhs) = do checkNumberComparisons _ _ = return () prop_checkSingleBracketOperators1 = verify checkSingleBracketOperators "[ test =~ foo ]" -checkSingleBracketOperators params (TC_Binary id SingleBracket "=~" lhs rhs) = - when (shellType params `elem` [Bash, Ksh]) $ +checkSingleBracketOperators params (TC_Binary id SingleBracket "=~" lhs rhs) + | shellType params `elem` [Bash, Ksh] = err id 2074 $ "Can't use =~ in [ ]. Use [[..]] instead." checkSingleBracketOperators _ _ = return () @@ -1165,10 +1164,10 @@ prop_checkGlobbedRegex5 = verifyNot checkGlobbedRegex "[[ $foo =~ \\* ]]" prop_checkGlobbedRegex6 = verifyNot checkGlobbedRegex "[[ $foo =~ (o*) ]]" prop_checkGlobbedRegex7 = verifyNot checkGlobbedRegex "[[ $foo =~ \\*foo ]]" prop_checkGlobbedRegex8 = verifyNot checkGlobbedRegex "[[ $foo =~ x\\* ]]" -checkGlobbedRegex _ (TC_Binary _ DoubleBracket "=~" _ rhs) = - let s = concat $ oversimplify rhs in - when (isConfusedGlobRegex s) $ - warn (getId rhs) 2049 "=~ is for regex, but this looks like a glob. Use = instead." +checkGlobbedRegex _ (TC_Binary _ DoubleBracket "=~" _ rhs) + | isConfusedGlobRegex s = + warn (getId rhs) 2049 "=~ is for regex, but this looks like a glob. Use = instead." + where s = concat $ oversimplify rhs checkGlobbedRegex _ _ = return () @@ -1512,8 +1511,8 @@ prop_checkIndirectExpansion2 = verifyNot checkIndirectExpansion "${foo//$n/lol}" prop_checkIndirectExpansion3 = verify checkIndirectExpansion "${$#}" prop_checkIndirectExpansion4 = verify checkIndirectExpansion "${var${n}_$((i%2))}" prop_checkIndirectExpansion5 = verifyNot checkIndirectExpansion "${bar}" -checkIndirectExpansion _ (T_DollarBraced i _ (T_NormalWord _ contents)) = - when (isIndirection contents) $ +checkIndirectExpansion _ (T_DollarBraced i _ (T_NormalWord _ contents)) + | isIndirection contents = err i 2082 "To expand via indirection, use arrays, ${!name} or (for sh only) eval." where isIndirection vars = @@ -1550,8 +1549,8 @@ checkInexplicablyUnquoted params (T_NormalWord id tokens) = mapM_ check (tails t case trapped of T_DollarExpansion id _ -> warnAboutExpansion id T_DollarBraced id _ _ -> warnAboutExpansion id - T_Literal id s -> - unless (quotesSingleThing a && quotesSingleThing b || isRegex (getPath (parentMap params) trapped)) $ + T_Literal id s + | not (quotesSingleThing a && quotesSingleThing b || isRegex (getPath (parentMap params) trapped)) -> warnAboutLiteral id _ -> return () @@ -1644,8 +1643,8 @@ checkSpuriousExec _ = doLists commentIfExec (T_Pipeline id _ list) = mapM_ commentIfExec $ take 1 list commentIfExec (T_Redirecting _ _ f@( - T_SimpleCommand id _ (cmd:arg:_))) = - when (f `isUnqualifiedCommand` "exec") $ + T_SimpleCommand id _ (cmd:arg:_))) + | f `isUnqualifiedCommand` "exec" = warn id 2093 "Remove \"exec \" if script should continue after this command." commentIfExec _ = return () @@ -1922,8 +1921,8 @@ prop_CheckVariableBraces3 = verifyNot checkVariableBraces "#shellcheck disable=S prop_CheckVariableBraces4 = verifyNot checkVariableBraces "echo $* $1" checkVariableBraces params t = case t of - T_DollarBraced id False _ -> - unless (name `elem` unbracedVariables) $ + T_DollarBraced id False _ + | name `notElem` unbracedVariables -> styleWithFix id 2250 "Prefer putting braces around variable references even when not strictly required." (fixFor t) @@ -2532,13 +2531,12 @@ checkUnpassedInFunctions params root = updateWith x@(name, _, _) = Map.insertWith (++) name [x] warnForGroup group = - when (all isArgumentless group) $ - -- Allow ignoring SC2120 on the function to ignore all calls - let (name, func) = getFunction group - ignoring = shouldIgnoreCode params 2120 func - in unless ignoring $ do - mapM_ suggestParams group - warnForDeclaration func name + -- Allow ignoring SC2120 on the function to ignore all calls + when (all isArgumentless group && not ignoring) $ do + mapM_ suggestParams group + warnForDeclaration func name + where (name, func) = getFunction group + ignoring = shouldIgnoreCode params 2120 func suggestParams (name, _, thing) = info (getId thing) 2119 $ @@ -2562,11 +2560,11 @@ prop_checkOverridingPath8 = verifyNot checkOverridingPath "PATH=$PATH:/stuff" checkOverridingPath _ (T_SimpleCommand _ vars []) = mapM_ checkVar vars where - checkVar (T_Assignment id Assign "PATH" [] word) = - let string = concat $ oversimplify word - in unless (any (`isInfixOf` string) ["/bin", "/sbin" ]) $ do + checkVar (T_Assignment id Assign "PATH" [] word) + | not $ any (`isInfixOf` string) ["/bin", "/sbin" ] = do when ('/' `elem` string && ':' `notElem` string) $ notify id when (isLiteral word && ':' `notElem` string && '/' `notElem` string) $ notify id + where string = concat $ oversimplify word checkVar _ = return () notify id = warn id 2123 "PATH is the shell search path. Use another name." checkOverridingPath _ _ = return () @@ -2577,8 +2575,8 @@ prop_checkTildeInPath3 = verifyNot checkTildeInPath "PATH=~/bin" checkTildeInPath _ (T_SimpleCommand _ vars _) = mapM_ checkVar vars where - checkVar (T_Assignment id Assign "PATH" [] (T_NormalWord _ parts)) = - when (any (\x -> isQuoted x && hasTilde x) parts) $ + checkVar (T_Assignment id Assign "PATH" [] (T_NormalWord _ parts)) + | any (\x -> isQuoted x && hasTilde x) parts = warn id 2147 "Literal tilde in PATH works poorly across programs." checkVar _ = return () @@ -2822,8 +2820,8 @@ prop_checkReadWithoutR3 = verifyNot checkReadWithoutR "read -t 0" prop_checkReadWithoutR4 = verifyNot checkReadWithoutR "read -t 0 && read --d '' -r bar" prop_checkReadWithoutR5 = verifyNot checkReadWithoutR "read -t 0 foo < file.txt" prop_checkReadWithoutR6 = verifyNot checkReadWithoutR "read -u 3 -t 0" -checkReadWithoutR _ t@T_SimpleCommand {} | t `isUnqualifiedCommand` "read" = - unless ("r" `elem` map snd flags || has_t0) $ +checkReadWithoutR _ t@T_SimpleCommand {} | t `isUnqualifiedCommand` "read" + && "r" `notElem` map snd flags && not has_t0 = info (getId $ getCommandTokenOrThis t) 2162 "read without -r will mangle backslashes." where flags = getAllFlags t @@ -2872,15 +2870,15 @@ checkUncheckedCdPushdPopd params root = [] else execWriter $ doAnalysis checkElement root where - checkElement t@T_SimpleCommand {} = do - let name = getName t - when(name `elem` ["cd", "pushd", "popd"] + checkElement t@T_SimpleCommand {} + | name `elem` ["cd", "pushd", "popd"] && not (isSafeDir t) && not (name `elem` ["pushd", "popd"] && ("n" `elem` map snd (getAllFlags t))) - && not (isCondition $ getPath (parentMap params) t)) $ + && not (isCondition $ getPath (parentMap params) t) = warnWithFix (getId t) 2164 ("Use '" ++ name ++ " ... || exit' or '" ++ name ++ " ... || return' in case " ++ name ++ " fails.") (fixWith [replaceEnd (getId t) params 0 " || exit"]) + where name = getName t checkElement _ = return () getName t = fromMaybe "" $ getCommandName t isSafeDir t = case oversimplify t of @@ -2953,10 +2951,10 @@ checkReturnAgainstZero _ token = case token of TC_Binary id _ _ lhs rhs -> check lhs rhs TA_Binary id _ lhs rhs -> check lhs rhs - TA_Unary id _ exp -> - when (isExitCode exp) $ message (getId exp) - TA_Sequence _ [exp] -> - when (isExitCode exp) $ message (getId exp) + TA_Unary id _ exp + | isExitCode exp -> message (getId exp) + TA_Sequence _ [exp] + | isExitCode exp -> message (getId exp) _ -> return () where check lhs rhs = @@ -3191,8 +3189,8 @@ prop_checkGlobAsCommand1 = verify checkGlobAsCommand "foo*" prop_checkGlobAsCommand2 = verify checkGlobAsCommand "$(var[i])" prop_checkGlobAsCommand3 = verifyNot checkGlobAsCommand "echo foo*" checkGlobAsCommand _ t = case t of - T_SimpleCommand _ _ (first:_) -> - when (isGlob first) $ + T_SimpleCommand _ _ (first:_) + | isGlob first -> warn (getId first) 2211 "This is a glob used as a command name. Was it supposed to be in ${..}, array, or is it missing quoting?" _ -> return () @@ -3202,8 +3200,8 @@ prop_checkFlagAsCommand2 = verify checkFlagAsCommand "foo\n --bar=baz" prop_checkFlagAsCommand3 = verifyNot checkFlagAsCommand "'--myexec--' args" prop_checkFlagAsCommand4 = verifyNot checkFlagAsCommand "var=cmd --arg" -- Handled by SC2037 checkFlagAsCommand _ t = case t of - T_SimpleCommand _ [] (first:_) -> - when (isUnquotedFlag first) $ + T_SimpleCommand _ [] (first:_) + | isUnquotedFlag first -> warn (getId first) 2215 "This flag is used as a command name. Bad line break or missing [ .. ]?" _ -> return () @@ -3227,7 +3225,7 @@ checkPipeToNowhere :: Parameters -> Token -> WriterT [TokenComment] Identity () checkPipeToNowhere _ t = case t of T_Pipeline _ _ (first:rest) -> mapM_ checkPipe rest - T_Redirecting _ redirects cmd -> when (any redirectsStdin redirects) $ checkRedir cmd + T_Redirecting _ redirects cmd | any redirectsStdin redirects -> checkRedir cmd _ -> return () where checkPipe redir = sequence_ $ do @@ -3408,8 +3406,8 @@ prop_checkRedirectionToCommand2 = verifyNot checkRedirectionToCommand "ls > 'rm' prop_checkRedirectionToCommand3 = verifyNot checkRedirectionToCommand "ls > myfile" checkRedirectionToCommand _ t = case t of - T_IoFile _ _ (T_NormalWord id [T_Literal _ str]) | str `elem` commonCommands -> - unless (str == "file") $ -- This would be confusing + T_IoFile _ _ (T_NormalWord id [T_Literal _ str]) | str `elem` commonCommands + && str /= "file" -> -- This would be confusing warn id 2238 "Redirecting to/from command name instead of file. Did you want pipes/xargs (or quote to ignore)?" _ -> return () From 21ad4196dba4fd47275319ccf8bb3e2d7745567f Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 9 Feb 2020 21:08:32 -0500 Subject: [PATCH 264/763] Simplify findFunction --- src/ShellCheck/Analytics.hs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 5f19352..31213f8 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2488,12 +2488,10 @@ checkUnpassedInFunctions params root = map (\t@(T_Function _ _ _ name _) -> (name,t)) functions functions = execWriter $ doAnalysis (tell . maybeToList . findFunction) root - findFunction t@(T_Function id _ _ name body) = - let flow = getVariableFlow params body - in - if any (isPositionalReference t) flow && not (any isPositionalAssignment flow) - then return t - else Nothing + findFunction t@(T_Function id _ _ name body) + | any (isPositionalReference t) flow && not (any isPositionalAssignment flow) + = return t + where flow = getVariableFlow params body findFunction _ = Nothing isPositionalAssignment x = From 43c24cf79c6629799c76bdfa35ef957da193fbe2 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 9 Feb 2020 21:14:52 -0500 Subject: [PATCH 265/763] Use Map.! instead of reimplementing it --- src/ShellCheck/Analytics.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 31213f8..1806b7d 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2544,7 +2544,7 @@ checkUnpassedInFunctions params root = name ++ " references arguments, but none are ever passed." getFunction ((name, _, _):_) = - (name, fromJust $ Map.lookup name functionMap) + (name, functionMap Map.! name) prop_checkOverridingPath1 = verify checkOverridingPath "PATH=\"$var/$foo\"" From 292b0840d9598d64cd5097444fb47ef1b7825004 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 9 Feb 2020 21:39:02 -0500 Subject: [PATCH 266/763] Simplify a double negative --- src/ShellCheck/Analytics.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 1806b7d..4d5f460 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2588,7 +2588,7 @@ prop_checkUnsupported3 = verify checkUnsupported "#!/bin/sh\ncase foo in bar) ba prop_checkUnsupported4 = verify checkUnsupported "#!/bin/ksh\ncase foo in bar) baz ;;& esac" prop_checkUnsupported5 = verify checkUnsupported "#!/bin/bash\necho \"${ ls; }\"" checkUnsupported params t = - when (not (null support) && (shellType params `notElem` support)) $ + unless (null support || (shellType params `elem` support)) $ report name where (name, support) = shellSupport t From 8e9290badba48894fa54af9128d63a83a2a23293 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 9 Feb 2020 21:40:05 -0500 Subject: [PATCH 267/763] Do toLower earlier --- src/ShellCheck/Analytics.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 4d5f460..aed0e22 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2594,7 +2594,7 @@ checkUnsupported params t = (name, support) = shellSupport t report s = err (getId t) 2127 $ "To use " ++ s ++ ", specify #!/usr/bin/env " ++ - (map toLower . intercalate " or " . map show $ support) + (intercalate " or " . map (map toLower . show) $ support) -- TODO: Move more of these checks here shellSupport t = From a223a7a5a58eb8f380faea6743d7ced3cf7b6cfc Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 9 Feb 2020 21:50:40 -0500 Subject: [PATCH 268/763] Remove unnecessary fromMaybes --- src/ShellCheck/Analytics.hs | 5 ++--- src/ShellCheck/AnalyzerLib.hs | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index aed0e22..f275229 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2823,11 +2823,10 @@ checkReadWithoutR _ t@T_SimpleCommand {} | t `isUnqualifiedCommand` "read" info (getId $ getCommandTokenOrThis t) 2162 "read without -r will mangle backslashes." where flags = getAllFlags t - has_t0 = fromMaybe False $ do + has_t0 = Just "0" == do parsed <- getOpts flagsForRead flags t <- lookup "t" parsed - str <- getLiteralString t - return $ str == "0" + getLiteralString t checkReadWithoutR _ _ = return () diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 69eaf63..5ff1c7d 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -454,7 +454,7 @@ leadType params t = T_BatsTest {} -> SubshellScope "@bats test" T_CoProcBody _ _ -> SubshellScope "coproc" T_Redirecting {} -> - if fromMaybe False causesSubshell + if causesSubshell == Just True then SubshellScope "pipeline" else NoneScope _ -> NoneScope From 962fad038c194006e8b1db2efb875c9d49f4daf8 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 9 Feb 2020 22:02:11 -0500 Subject: [PATCH 269/763] Avoid a zip that breaks fusion --- src/ShellCheck/Analytics.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index f275229..61b8515 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -3088,7 +3088,7 @@ checkUnmatchableCases params t = return $ warn (getId candidate) 2195 "This pattern will never match the case statement's word. Double check them." - tupMap f l = zip l (map f l) + tupMap f l = map (\x -> (x, f x)) l checkDoms ((glob, Just x), rest) = case filter (\(_, p) -> x `pseudoGlobIsSuperSetof` p) valids of ((first,_):_) -> do From 7fc94963204b85547918d48d458e67530a530227 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 9 Feb 2020 22:08:31 -0500 Subject: [PATCH 270/763] Use forM_ instead of reimplementing it --- src/ShellCheck/Analytics.hs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 61b8515..794c3f5 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -3090,11 +3090,10 @@ checkUnmatchableCases params t = tupMap f l = map (\x -> (x, f x)) l checkDoms ((glob, Just x), rest) = - case filter (\(_, p) -> x `pseudoGlobIsSuperSetof` p) valids of - ((first,_):_) -> do + forM_ (find (\(_, p) -> x `pseudoGlobIsSuperSetof` p) valids) $ + \(first,_) -> do warn (getId glob) 2221 $ "This pattern always overrides a later one" <> patternContext (getId first) warn (getId first) 2222 $ "This pattern never matches because of a previous pattern" <> patternContext (getId glob) - _ -> return () where patternContext :: Id -> String patternContext id = From 8f0448133ca7437f704e584c018519e2ac3b033c Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 9 Feb 2020 22:14:44 -0500 Subject: [PATCH 271/763] Use isNothing instead of reimplementing it --- src/ShellCheck/Analytics.hs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 794c3f5..a082753 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -3250,9 +3250,8 @@ checkPipeToNowhere _ t = "Redirecting to '" ++ name ++ "', a command that doesn't read stdin. " ++ suggestion -- Could any words in a SimpleCommand consume stdin (e.g. echo "$(cat)")? - hasAdditionalConsumers t = fromMaybe True $ do + hasAdditionalConsumers t = isNothing $ doAnalysis (guard . not . mayConsume) t - return False mayConsume t = case t of From ea24e25efd79ff893c540269e9747d1a662640ea Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 9 Feb 2020 22:22:32 -0500 Subject: [PATCH 272/763] Use Map.member instead of isJust and Map.lookup --- src/ShellCheck/Analytics.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index a082753..aabb22b 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -849,7 +849,7 @@ checkArrayWithoutIndex params _ = readF _ _ _ = return [] writeF _ (T_Assignment id mode name [] _) _ (DataString _) = do - isArray <- gets (isJust . Map.lookup name) + isArray <- gets (Map.member name) return $ if not isArray then [] else case mode of Assign -> [makeComment WarningC id 2178 "Variable was used as an array but is now assigned a string."] From c95914f9b3326d92748017b7777e23767c8b955a Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 9 Feb 2020 22:41:02 -0500 Subject: [PATCH 273/763] Simplify determineShell --- src/ShellCheck/AnalyzerLib.hs | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 5ff1c7d..212e7bc 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -239,19 +239,15 @@ prop_determineShell8 = determineShellTest' (Just Ksh) "#!/bin/sh" == Sh determineShellTest = determineShellTest' Nothing determineShellTest' fallbackShell = determineShell fallbackShell . fromJust . prRoot . pScript -determineShell fallbackShell t = fromMaybe Bash $ do - shellString <- foldl mplus Nothing $ getCandidates t +determineShell fallbackShell t = fromMaybe Bash $ shellForExecutable shellString `mplus` fallbackShell where - forAnnotation t = - case t of - (ShellOverride s) -> return s - _ -> fail "" - getCandidates :: Token -> [Maybe String] - getCandidates t@T_Script {} = [Just $ fromShebang t] - getCandidates (T_Annotation _ annotations s) = - map forAnnotation annotations ++ - [Just $ fromShebang s] + shellString = getCandidate t + getCandidate :: Token -> String + getCandidate t@T_Script {} = fromShebang t + getCandidate (T_Annotation _ annotations s) = + fromMaybe (fromShebang s) $ + listToMaybe [s | ShellOverride s <- annotations] fromShebang (T_Script _ (T_Literal _ s) _) = executableFromShebang s -- Given a string like "/bin/bash" or "/usr/bin/env dash", From 6d06103cab35e3179046cedcda1fd6ea45555790 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 9 Feb 2020 22:51:10 -0500 Subject: [PATCH 274/763] Remove unnecessary uses of head --- src/ShellCheck/Analytics.hs | 2 +- src/ShellCheck/AnalyzerLib.hs | 9 ++++----- src/ShellCheck/Checks/Commands.hs | 4 ++-- src/ShellCheck/Checks/ShellSupport.hs | 4 ++-- src/ShellCheck/Parser.hs | 10 +++++----- 5 files changed, 14 insertions(+), 15 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index aabb22b..ffa04c8 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2293,7 +2293,7 @@ checkWhileReadPitfalls _ (T_WhileExpression id [command] contents) isStdinReadCommand (T_Pipeline _ _ [T_Redirecting id redirs cmd]) = let plaintext = oversimplify cmd - in head (plaintext ++ [""]) == "read" + in headOrDefault "" plaintext == "read" && ("-u" `notElem` plaintext) && all (not . stdinRedirect) redirs isStdinReadCommand _ = False diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 212e7bc..364722b 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -255,7 +255,7 @@ determineShell fallbackShell t = fromMaybe Bash $ executableFromShebang :: String -> String executableFromShebang = shellFor where - shellFor s | "/env " `isInfixOf` s = head (drop 1 (words s)++[""]) + shellFor s | "/env " `isInfixOf` s = headOrDefault "" (drop 1 $ words s) shellFor s | ' ' `elem` s = shellFor $ takeWhile (/= ' ') s shellFor s = reverse . takeWhile (/= '/') . reverse $ s @@ -295,7 +295,7 @@ isQuoteFree = isQuoteFreeNode False isQuoteFreeNode strict tree t = (isQuoteFreeElement t == Just True) || - head (mapMaybe isQuoteFreeContext (drop 1 $ getPath tree t) ++ [False]) + headOrDefault False (mapMaybe isQuoteFreeContext (drop 1 $ getPath tree t)) where -- Is this node self-quoting in itself? isQuoteFreeElement t = @@ -758,9 +758,8 @@ getReferencedVariables parents t = _ -> Nothing getIfReference context token = maybeToList $ do - str <- getLiteralStringExt literalizer token - guard . not $ null str - when (isDigit $ head str) $ fail "is a number" + str@(h:_) <- getLiteralStringExt literalizer token + when (isDigit h) $ fail "is a number" return (context, token, getBracedReference str) isDereferencing = (`elem` ["-eq", "-ne", "-lt", "-le", "-gt", "-ge"]) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 97de0f6..3407030 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -279,10 +279,10 @@ checkGrepRe = CommandCheck (Basename "grep") check where grepGlobFlags = ["fixed-strings", "F", "include", "exclude", "exclude-dir", "o", "only-matching"] wordStartingWith c = - head . filter ([c] `isPrefixOf`) $ candidates + headOrDefault (c:"test") . filter ([c] `isPrefixOf`) $ candidates where candidates = - sampleWords ++ map (\(x:r) -> toUpper x : r) sampleWords ++ [c:"test"] + sampleWords ++ map (\(x:r) -> toUpper x : r) sampleWords getSuspiciousRegexWildcard str = if not $ str `matches` contra diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 4e0655d..924b002 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -457,8 +457,8 @@ checkEchoSed = ForShell [Bash, Ksh] f -- This should have used backreferences, but TDFA doesn't support them sedRe = mkRegex "^s(.)([^\n]*)g?$" isSimpleSed s = fromMaybe False $ do - [first,rest] <- matchRegex sedRe s - let delimiters = filter (== head first) rest + [h:_,rest] <- matchRegex sedRe s + let delimiters = filter (== h) rest guard $ length delimiters == 2 return True checkIn id s = diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 30529e0..f37a56a 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -186,12 +186,12 @@ getNextIdSpanningTokens startTok endTok = do -- Get an ID starting from the first token of the list, and ending after the last getNextIdSpanningTokenList list = - if null list - then do + case list of + [] -> do pos <- getPosition getNextIdBetween pos pos - else - getNextIdSpanningTokens (head list) (last list) + (h:_) -> + getNextIdSpanningTokens h (last list) -- Get the span covered by an id getSpanForId :: Monad m => Id -> SCParser m (SourcePos, SourcePos) @@ -1826,7 +1826,7 @@ readPendingHereDocs = do let thereIsNoTrailer = null trailingSpace && null trailer let leaderIsOk = null leadingSpace || dashed == Dashed && leadingSpacesAreTabs - let trailerStart = if null trailer then '\0' else head trailer + let trailerStart = case trailer of [] -> '\0'; (h:_) -> h let hasTrailingSpace = not $ null trailingSpace let hasTrailer = not $ null trailer let ppt = parseProblemAt trailerPos ErrorC From d5c51281158d0b29408b6aae5035c6b6ee700b5b Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 9 Feb 2020 23:06:12 -0500 Subject: [PATCH 275/763] Use isJust instead of reimplementing it --- src/ShellCheck/Checks/ShellSupport.hs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 924b002..83d7887 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -456,11 +456,10 @@ checkEchoSed = ForShell [Bash, Ksh] f -- This should have used backreferences, but TDFA doesn't support them sedRe = mkRegex "^s(.)([^\n]*)g?$" - isSimpleSed s = fromMaybe False $ do + isSimpleSed s = isJust $ do [h:_,rest] <- matchRegex sedRe s let delimiters = filter (== h) rest guard $ length delimiters == 2 - return True checkIn id s = when (isSimpleSed s) $ style id 2001 "See if you can use ${variable//search/replace} instead." From 42abcb7ae2a68b283f3592b13736d988bcfb51e5 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 9 Feb 2020 23:12:27 -0500 Subject: [PATCH 276/763] Simplify shellFromFilename --- src/ShellCheck/Checker.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Checker.hs b/src/ShellCheck/Checker.hs index 6370f75..60280ec 100644 --- a/src/ShellCheck/Checker.hs +++ b/src/ShellCheck/Checker.hs @@ -48,7 +48,7 @@ tokenToPosition startMap t = fromMaybe fail $ do where fail = error "Internal shellcheck error: id doesn't exist. Please report!" -shellFromFilename filename = foldl mplus Nothing candidates +shellFromFilename filename = listToMaybe candidates where shellExtensions = [(".ksh", Ksh) ,(".bash", Bash) @@ -57,7 +57,7 @@ shellFromFilename filename = foldl mplus Nothing candidates -- The `.sh` is too generic to determine the shell: -- We fallback to Bash in this case and emit SC2148 if there is no shebang candidates = - map (\(ext,sh) -> if ext `isSuffixOf` filename then Just sh else Nothing) shellExtensions + [sh | (ext,sh) <- shellExtensions, ext `isSuffixOf` filename] checkScript :: Monad m => SystemInterface m -> CheckSpec -> m CheckResult checkScript sys spec = do From 85c49a8af9aed6e26604ea6756ed783b6ca9ee90 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 9 Feb 2020 23:50:48 -0500 Subject: [PATCH 277/763] Simplify mockedSystemInterface --- src/ShellCheck/Interface.hs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/Interface.hs b/src/ShellCheck/Interface.hs index aa12fc2..e51359e 100644 --- a/src/ShellCheck/Interface.hs +++ b/src/ShellCheck/Interface.hs @@ -316,10 +316,10 @@ mockedSystemInterface files = SystemInterface { siGetConfig = const $ return Nothing } where - rf file = - case filter ((== file) . fst) files of - [] -> return $ Left "File not included in mock." - [(_, contents)] -> return $ Right contents + rf file = return $ + case find ((== file) . fst) files of + Nothing -> Left "File not included in mock." + Just (_, contents) -> Right contents fs _ _ file = return file mockRcFile rcfile mock = mock { From 440d0038aaa3eacaf7bbb7b9bbe55b7c37663063 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Tue, 11 Feb 2020 01:03:10 -0500 Subject: [PATCH 278/763] Remove a partial pattern match equivalent to fromJust from checkFindNameGlob --- src/ShellCheck/Checks/Commands.hs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 3beca25..f8f3a20 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -197,13 +197,11 @@ prop_checkFindNameGlob1 = verify checkFindNameGlob "find / -name *.php" prop_checkFindNameGlob2 = verify checkFindNameGlob "find / -type f -ipath *(foo)" prop_checkFindNameGlob3 = verifyNot checkFindNameGlob "find * -name '*.php'" checkFindNameGlob = CommandCheck (Basename "find") (f . arguments) where - acceptsGlob (Just s) = s `elem` [ "-ilname", "-iname", "-ipath", "-iregex", "-iwholename", "-lname", "-name", "-path", "-regex", "-wholename" ] - acceptsGlob _ = False + acceptsGlob s = s `elem` [ "-ilname", "-iname", "-ipath", "-iregex", "-iwholename", "-lname", "-name", "-path", "-regex", "-wholename" ] f [] = return () f [x] = return () f (a:b:r) = do - when (acceptsGlob (getLiteralString a) && isGlob b) $ do - let (Just s) = getLiteralString a + forM_ (getLiteralString a) $ \s -> when (acceptsGlob s && isGlob b) $ warn (getId b) 2061 $ "Quote the parameter to " ++ s ++ " so the shell won't interpret it." f (b:r) From eecd003e2db0776ed989c0f04df9bfba75689d11 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Tue, 11 Feb 2020 01:04:49 -0500 Subject: [PATCH 279/763] Optimize patterns in checkFindNameGlob 1. Instead of pattern-matching the same list multiple times, do it only once and then pass the pieces separately. 2. Don't reconstruct an object equivalent to one we just deconstructed. --- src/ShellCheck/Checks/Commands.hs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index f8f3a20..e321102 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -199,11 +199,12 @@ prop_checkFindNameGlob3 = verifyNot checkFindNameGlob "find * -name '*.php'" checkFindNameGlob = CommandCheck (Basename "find") (f . arguments) where acceptsGlob s = s `elem` [ "-ilname", "-iname", "-ipath", "-iregex", "-iwholename", "-lname", "-name", "-path", "-regex", "-wholename" ] f [] = return () - f [x] = return () - f (a:b:r) = do + f (x:xs) = g x xs + g _ [] = return () + g a (b:r) = do forM_ (getLiteralString a) $ \s -> when (acceptsGlob s && isGlob b) $ warn (getId b) 2061 $ "Quote the parameter to " ++ s ++ " so the shell won't interpret it." - f (b:r) + g b r prop_checkNeedlessExpr = verify checkNeedlessExpr "foo=$(expr 3 + 2)" From 472579052b324613aa56c2060e7205d1d9295130 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 15 Feb 2020 16:56:20 -0800 Subject: [PATCH 280/763] Don't try to deploy on PRs --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 6c601a6..ff057a4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,6 +15,7 @@ jobs: os: osx - stage: Deploy docker image + if: branch = master script: - source ./.multi_arch_docker - set -ex; multi_arch_docker::main; set +x From 1da0becb0fe46fc7bb1f26f083d658a2be671a1c Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 15 Feb 2020 19:22:27 -0800 Subject: [PATCH 281/763] Rename 'Test' stage --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ff057a4..6a6c435 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,9 @@ services: jobs: include: - - stage: Test + - stage: Build binaries + + # This must weirdly not have a dash, otherwise an empty job is created env: BUILD=linux - env: BUILD=windows - env: BUILD=armv6hf From 106f321cf0cceef51f806a74d51f91b7c4da6744 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 17 Feb 2020 11:13:29 -0800 Subject: [PATCH 282/763] Parse keywords with case sensitivity (fixes #1809) --- CHANGELOG.md | 3 ++- src/ShellCheck/Parser.hs | 9 ++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0066a34..5931276 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,8 @@ - SC2256: Warn about translated strings that are known variables ### Changed -- SC2230: This check is now off by default +- SC2230: 'command -v' suggestion is now off by default (-i deprecate-which) +- SC1081: Keywords are now correctly parsed case sensitively, with a warning ## v0.7.0 - 2019-07-28 ### Added diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 67de013..d5fe02e 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -2304,6 +2304,7 @@ prop_readIfClause2 = isWarning readIfClause "if false; then; echo oo; fi" prop_readIfClause3 = isWarning readIfClause "if false; then true; else; echo lol; fi" prop_readIfClause4 = isWarning readIfClause "if false; then true; else if true; then echo lol; fi; fi" prop_readIfClause5 = isOk readIfClause "if false; then true; else\nif true; then echo lol; fi; fi" +prop_readIfClause6 = isWarning readIfClause "if true\nthen\nDo the thing\nfi" readIfClause = called "if expression" $ do start <- startSpan pos <- getPosition @@ -2890,6 +2891,7 @@ redirToken c t = try $ do tryWordToken s t = tryParseWordToken s t `thenSkip` spacing tryParseWordToken keyword t = try $ do + pos <- getPosition start <- startSpan str <- anycaseString keyword id <- endSpan start @@ -2905,9 +2907,10 @@ tryParseWordToken keyword t = try $ do _ -> return () lookAhead keywordSeparator - when (str /= keyword) $ - parseProblem ErrorC 1081 $ - "Scripts are case sensitive. Use '" ++ keyword ++ "', not '" ++ str ++ "'." + when (str /= keyword) $ do + parseProblemAt pos ErrorC 1081 $ + "Scripts are case sensitive. Use '" ++ keyword ++ "', not '" ++ str ++ "' (or quote if literal)." + fail "" return $ t id anycaseString = From 99d6df8a0840ef424a19b6cb25bf288aae30da02 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 17 Feb 2020 12:27:24 -0800 Subject: [PATCH 283/763] Bump SC1102/SC1105 about ambiguous `$((` to Error (fixes #1836) --- src/ShellCheck/Parser.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index d5fe02e..64d75c6 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -1554,7 +1554,7 @@ readDollarExpression = do readDollarExp = arithmetic <|> readDollarExpansion <|> readDollarBracket <|> readDollarBraceCommandExpansion <|> readDollarBraced <|> readDollarVariable where arithmetic = readAmbiguous "$((" readDollarArithmetic readDollarExpansion (\pos -> - parseNoteAt pos WarningC 1102 "Shells disambiguate $(( differently or not at all. For $(command substition), add space after $( . For $((arithmetics)), fix parsing errors.") + parseNoteAt pos ErrorC 1102 "Shells disambiguate $(( differently or not at all. For $(command substition), add space after $( . For $((arithmetics)), fix parsing errors.") prop_readDollarSingleQuote = isOk readDollarSingleQuote "$'foo\\\'lol'" readDollarSingleQuote = called "$'..' expression" $ do @@ -2673,7 +2673,7 @@ readCompoundCommand = do cmd <- choice [ readBraceGroup, readAmbiguous "((" readArithmeticExpression readSubshell (\pos -> - parseNoteAt pos WarningC 1105 "Shells disambiguate (( differently or not at all. For subshell, add spaces around ( . For ((, fix parsing errors."), + parseNoteAt pos ErrorC 1105 "Shells disambiguate (( differently or not at all. For subshell, add spaces around ( . For ((, fix parsing errors."), readSubshell, readCondition, readWhileClause, From a75219e5256b0cb4c339ed4f259b9a49269a5663 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 17 Feb 2020 12:44:38 -0800 Subject: [PATCH 284/763] Remove unused instance Ord Replacement (fixes #1829) --- src/ShellCheck/Interface.hs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/ShellCheck/Interface.hs b/src/ShellCheck/Interface.hs index e51359e..85d25c0 100644 --- a/src/ShellCheck/Interface.hs +++ b/src/ShellCheck/Interface.hs @@ -256,9 +256,6 @@ data Replacement = Replacement { data InsertionPoint = InsertBefore | InsertAfter deriving (Show, Eq, Generic, NFData) -instance Ord Replacement where - compare r1 r2 = (repStartPos r1) `compare` (repStartPos r2) - newReplacement = Replacement { repStartPos = newPosition, repEndPos = newPosition, From 4c9210af7916f78bc041ca7871b868a266b38d5d Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 17 Feb 2020 14:20:21 -0800 Subject: [PATCH 285/763] Inspect 'alias' commands for referenced variables (Fixes #1832) --- src/ShellCheck/Analytics.hs | 1 + src/ShellCheck/AnalyzerLib.hs | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 3d00d29..e695d24 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2127,6 +2127,7 @@ prop_checkUnused43= verifyTree checkUnusedAssignments "DEFINE_string foo '' ''" prop_checkUnused44= verifyNotTree checkUnusedAssignments "DEFINE_string \"foo$ibar\" x y" prop_checkUnused45= verifyTree checkUnusedAssignments "readonly foo=bar" prop_checkUnused46= verifyTree checkUnusedAssignments "readonly foo=(bar)" +prop_checkUnused47= verifyNotTree checkUnusedAssignments "a=1; alias hello='echo $a'" checkUnusedAssignments params t = execWriter (mapM_ warnFor unused) where flow = variableFlow params diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index dbf59a0..42f57b1 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -544,8 +544,9 @@ getReferencedVariableCommand base@(T_SimpleCommand _ _ (T_NormalWord _ (T_Litera else [] "trap" -> case rest of - head:_ -> map (\x -> (head, head, x)) $ getVariablesFromLiteralToken head + head:_ -> map (\x -> (base, head, x)) $ getVariablesFromLiteralToken head _ -> [] + "alias" -> [(base, token, name) | token <- rest, name <- getVariablesFromLiteralToken token] _ -> [] where getReference t@(T_Assignment _ _ name _ value) = [(t, t, name)] From 7b998239afe21e89c1c57e7f26947fb4b640214f Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 17 Feb 2020 18:02:23 -0800 Subject: [PATCH 286/763] SC2257: Warn when changing arithmetic variables in redirections --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5931276..11860ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ - SC2254: Suggest quoting expansions in case statements - SC2255: Suggest using `$((..))` in `[ 2*3 -eq 6 ]` - SC2256: Warn about translated strings that are known variables +- SC2257: Warn about arithmetic mutation in redirections ### Changed - SC2230: 'command -v' suggestion is now off by default (-i deprecate-which) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index e695d24..df3d3f4 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -189,6 +189,7 @@ nodeChecks = [ ,checkDollarQuoteParen ,checkUselessBang ,checkTranslatedStringVariable + ,checkModifiedArithmeticInRedirection ] optionalChecks = map fst optionalTreeChecks @@ -3530,5 +3531,40 @@ checkUselessBang params t = when (hasSetE params) $ mapM_ check (getNonReturning x:rest -> x : dropLast rest _ -> [] +prop_checkModifiedArithmeticInRedirection1 = verify checkModifiedArithmeticInRedirection "ls > $((i++))" +prop_checkModifiedArithmeticInRedirection2 = verify checkModifiedArithmeticInRedirection "cat < \"foo$((i++)).txt\"" +prop_checkModifiedArithmeticInRedirection3 = verifyNot checkModifiedArithmeticInRedirection "while true; do true; done > $((i++))" +prop_checkModifiedArithmeticInRedirection4 = verify checkModifiedArithmeticInRedirection "cat <<< $((i++))" +prop_checkModifiedArithmeticInRedirection5 = verify checkModifiedArithmeticInRedirection "cat << foo\n$((i++))\nfoo\n" +checkModifiedArithmeticInRedirection _ t = + case t of + T_Redirecting _ redirs (T_SimpleCommand _ _ (_:_)) -> mapM_ checkRedirs redirs + _ -> return () + where + checkRedirs t = + case t of + T_FdRedirect _ _ (T_IoFile _ _ word) -> + mapM_ checkArithmetic $ getWordParts word + T_FdRedirect _ _ (T_HereString _ word) -> + mapM_ checkArithmetic $ getWordParts word + T_FdRedirect _ _ (T_HereDoc _ _ _ _ list) -> + mapM_ checkArithmetic list + _ -> return () + checkArithmetic t = + case t of + T_DollarArithmetic _ x -> checkModifying x + _ -> return () + checkModifying t = + case t of + TA_Sequence _ list -> mapM_ checkModifying list + TA_Unary id s _ | s `elem` ["|++", "++|", "|--", "--|"] -> warnFor id + TA_Assignment id _ _ _ -> warnFor id + TA_Binary _ _ x y -> mapM_ checkModifying [x ,y] + TA_Trinary _ x y z -> mapM_ checkModifying [x, y, z] + _ -> return () + warnFor id = + warn id 2257 "Arithmetic modifications in command redirections may be discarded. Do them separately." + + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) From 00574dd1fce3b453de151cb53a73075b959fc59c Mon Sep 17 00:00:00 2001 From: Luke Davis Date: Tue, 3 Mar 2020 13:23:24 -0700 Subject: [PATCH 287/763] Add conda install instructions --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 5acfb89..a8ec54b 100644 --- a/README.md +++ b/README.md @@ -197,6 +197,10 @@ Or Windows (via [scoop](http://scoop.sh)): C:\> scoop install shellcheck ``` +From [conda-forge](https://anaconda.org/conda-forge/shellcheck): + + conda install -c conda-forge shellcheck + From Snap Store: snap install --channel=edge shellcheck From 9b66bc2f136216431cb1e01395a379750d3c1688 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 7 Mar 2020 16:16:47 -0800 Subject: [PATCH 288/763] Upload to assets to GitHub --- .github_deploy | 57 +++++++++++++++++++++++++++++++++++++++++++ .travis.yml | 5 ++-- Dockerfile.multi-arch | 4 +-- 3 files changed, 62 insertions(+), 4 deletions(-) create mode 100755 .github_deploy diff --git a/.github_deploy b/.github_deploy new file mode 100755 index 0000000..8f648ed --- /dev/null +++ b/.github_deploy @@ -0,0 +1,57 @@ +#!/bin/bash +set -x +shopt -s extglob + +if [[ "$TRAVIS_SECURE_ENV_VARS" != "true" ]] +then + echo >&2 "Missing TRAVIS_SECURE_ENV_VARS. Skipping GitHub deployment." + exit 0 +fi + +install_deps() { + version="2.7.0" # 2.14.1 fails to overwrite duplicates + case "$(uname)" in + Linux) + sudo apt-get update + sudo apt-get install curl + curl -L "https://github.com/github/hub/releases/download/v$version/hub-linux-amd64-$version.tgz" | tar xvz --strip-components=1 "hub-linux-amd64-$version/bin/hub" + ;; + Darwin) + curl -L "https://github.com/github/hub/releases/download/v$version/hub-darwin-amd64-$version.tgz" | tar xvz --strip-components=1 "hub-darwin-amd64-$version/bin/hub" + ;; + *) + echo "Unknown: $(uname)" + exit 1 + ;; + esac + + hub_path="$PWD/bin/hub" + hub() { + "$hub_path" "$@" + } +} +install_deps + +export EDITOR="touch" + +# Sanity check +hub release show latest || exit 1 + +for tag in $TAGS +do + if ! hub release show "$tag" + then + echo "Creating new release $tag" + git show --no-patch --format='format:%B' > description + hub release create -F description "$tag" + fi + + files=() + for file in deploy/* + do + [[ $file == *.@(xz|gz|zip) ]] || continue + files+=(-a "$file") + done + hub release edit "${files[@]}" "$tag" || exit 1 +done + diff --git a/.travis.yml b/.travis.yml index 6a6c435..ac53f0c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,7 +25,7 @@ jobs: before_install: | DOCKER_BASE="$DOCKER_USERNAME/shellcheck" DOCKER_BUILDS="" - TAGS="" + export TAGS="" test "$TRAVIS_BRANCH" = master && TAGS="$TAGS latest" || true test -n "$TRAVIS_TAG" && TAGS="$TAGS stable $TRAVIS_TAG" || true echo "Tags are $TAGS" @@ -36,6 +36,7 @@ script: - ./striptests - set -ex; build_"$BUILD"; set +x; - ./.prepare_deploy + - ./.github_deploy after_failure: | id @@ -54,5 +55,5 @@ deploy: local_dir: deploy on: repo: koalaman/shellcheck - condition: $TRAVIS_BUILD_STAGE_NAME = Test + condition: $TRAVIS_BUILD_STAGE_NAME = "Build binaries" all_branches: true diff --git a/Dockerfile.multi-arch b/Dockerfile.multi-arch index 193e762..217aa74 100644 --- a/Dockerfile.multi-arch +++ b/Dockerfile.multi-arch @@ -11,8 +11,8 @@ RUN set -x; \ if [ "${arch}" = 'armv7l' ]; then \ arch='armv6hf'; \ fi; \ - url_base='https://shellcheck.storage.googleapis.com/'; \ - tar_file="shellcheck-${tag}.linux.${arch}.tar.xz"; \ + url_base='https://github.com/koalaman/shellcheck/releases/download/'; \ + tar_file="${tag}/shellcheck-${tag}.linux.${arch}.tar.xz"; \ wget "${url_base}${tar_file}" -O - | tar xJf -; \ mv "shellcheck-${tag}/shellcheck" /bin/; \ rm -rf "shellcheck-${tag}"; \ From 741d499b3d3e7a6fed524b86d5ff0f9ebd0e353f Mon Sep 17 00:00:00 2001 From: Austin English Date: Sat, 7 Mar 2020 19:23:35 -0600 Subject: [PATCH 289/763] src/ShellCheck/Analytics.hs: suggest using a shell directive for SC2148 --- src/ShellCheck/Analytics.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index df3d3f4..5ad51d7 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -559,7 +559,7 @@ checkShebang params (T_Annotation _ list t) = checkShebang params (T_Script _ (T_Literal id sb) _) = execWriter $ do unless (shellTypeSpecified params) $ do when (null sb) $ - err id 2148 "Tips depend on target shell and yours is unknown. Add a shebang." + err id 2148 "Tips depend on target shell and yours is unknown. Add a shebang or a 'shell' directive." when (executableFromShebang sb == "ash") $ warn id 2187 "Ash scripts will be checked as Dash. Add '# shellcheck shell=dash' to silence." unless (null sb) $ do From 014a66f3f6b2f8edd813ef8d03c5c860def69801 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 7 Mar 2020 17:49:10 -0800 Subject: [PATCH 290/763] Fix TravisCI condition --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index ac53f0c..428fc8e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ services: jobs: include: - - stage: Build binaries + - stage: Build # This must weirdly not have a dash, otherwise an empty job is created env: BUILD=linux @@ -55,5 +55,4 @@ deploy: local_dir: deploy on: repo: koalaman/shellcheck - condition: $TRAVIS_BUILD_STAGE_NAME = "Build binaries" all_branches: true From 68a03e05e5e030d7274c712ffa39768310e70f8a Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 8 Mar 2020 17:41:46 -0700 Subject: [PATCH 291/763] Refer to GitHub rather than GCS for release builds --- README.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 5acfb89..ce57d07 100644 --- a/README.md +++ b/README.md @@ -217,13 +217,14 @@ nix-env -iA nixpkgs.shellcheck Alternatively, you can download pre-compiled binaries for the latest release here: -* [Linux, x86_64](https://storage.googleapis.com/shellcheck/shellcheck-stable.linux.x86_64.tar.xz) (statically linked) -* [Linux, armv6hf](https://storage.googleapis.com/shellcheck/shellcheck-stable.linux.armv6hf.tar.xz), i.e. Raspberry Pi (statically linked) -* [Linux, aarch64](https://storage.googleapis.com/shellcheck/shellcheck-stable.linux.aarch64.tar.xz) aka ARM64 (statically linked) -* [MacOS, x86_64](https://shellcheck.storage.googleapis.com/shellcheck-stable.darwin.x86_64.tar.xz) -* [Windows, x86](https://storage.googleapis.com/shellcheck/shellcheck-stable.zip) +* [Linux, x86_64](https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.x86_64.tar.xz) (statically linked) +* [Linux, armv6hf](https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.armv6hf.tar.xz), i.e. Raspberry Pi (statically linked) +* [Linux, aarch64](https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.aarch64.tar.xz) aka ARM64 (statically linked) +* [MacOS, x86_64](https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.darwin.x86_64.tar.xz) +* [Windows, x86](https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.zip) -or see the [storage bucket listing](https://shellcheck.storage.googleapis.com/index.html) for checksums, older versions and the latest daily builds. +or see the [GitHub Releases](https://github.com/koalaman/shellcheck/releases) for other releases +(including the [latest](https://github.com/koalaman/shellcheck/releases/tag/latest) meta-release for daily git builds). Distro packages already come with a `man` page. If you are building from source, it can be installed with: @@ -250,7 +251,7 @@ A simple installer may do something like: ```bash scversion="stable" # or "v0.4.7", or "latest" -wget -qO- "https://storage.googleapis.com/shellcheck/shellcheck-${scversion?}.linux.x86_64.tar.xz" | tar -xJv +wget -qO- "https://github.com/koalaman/shellcheck/releases/download/${scversion?}/shellcheck-${scversion?}.linux.x86_64.tar.xz" | tar -xJv cp "shellcheck-${scversion}/shellcheck" /usr/bin/ shellcheck --version ``` From 45a67e7c64b3ac00a05a35dc82d77826165bc43d Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Tue, 10 Mar 2020 13:27:52 -0400 Subject: [PATCH 292/763] Use headOrDefault instead of fromMaybe and listToMaybe --- src/ShellCheck/AnalyzerLib.hs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 42f57b1..5f33b30 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -246,8 +246,7 @@ determineShell fallbackShell t = fromMaybe Bash $ getCandidate :: Token -> String getCandidate t@T_Script {} = fromShebang t getCandidate (T_Annotation _ annotations s) = - fromMaybe (fromShebang s) $ - listToMaybe [s | ShellOverride s <- annotations] + headOrDefault (fromShebang s) [s | ShellOverride s <- annotations] fromShebang (T_Script _ (T_Literal _ s) _) = executableFromShebang s -- Given a string like "/bin/bash" or "/usr/bin/env dash", @@ -852,7 +851,7 @@ getBracedReference s = fromMaybe s $ prop_getBracedModifier1 = getBracedModifier "foo:bar:baz" == ":bar:baz" prop_getBracedModifier2 = getBracedModifier "!var:-foo" == ":-foo" prop_getBracedModifier3 = getBracedModifier "foo[bar]" == "[bar]" -getBracedModifier s = fromMaybe "" . listToMaybe $ do +getBracedModifier s = headOrDefault "" $ do let var = getBracedReference s a <- dropModifier s dropPrefix var a From c43b19f89789e9ca92da1674fec0e3e5b6e9ffd2 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 14 Mar 2020 21:15:47 -0700 Subject: [PATCH 293/763] Make SC2095 (ssh in while read loops) more robust and suggest fixes --- src/ShellCheck/ASTLib.hs | 17 +++++++--- src/ShellCheck/Analytics.hs | 68 ++++++++++++++++++++++++++++--------- 2 files changed, 65 insertions(+), 20 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index ab28959..bc97404 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -302,6 +302,11 @@ getCommand t = getCommandName :: Token -> Maybe String getCommandName = fst . getCommandNameAndToken +-- Maybe get the name+arguments of a command. +getCommandArgv t = do + (T_SimpleCommand _ _ args@(_:_)) <- getCommand t + return args + -- Get the command name token from a command, i.e. -- the token representing 'ls' in 'ls -la 2> foo'. -- If it can't be determined, return the original token. @@ -367,19 +372,23 @@ isFunctionLike t = isBraceExpansion t = case t of T_BraceExpansion {} -> True; _ -> False -- Get the lists of commands from tokens that contain them, such as --- the body of while loops or branches of if statements. +-- the conditions and bodies of while loops or branches of if statements. getCommandSequences :: Token -> [[Token]] getCommandSequences t = case t of T_Script _ _ cmds -> [cmds] T_BraceGroup _ cmds -> [cmds] T_Subshell _ cmds -> [cmds] - T_WhileExpression _ _ cmds -> [cmds] - T_UntilExpression _ _ cmds -> [cmds] + T_WhileExpression _ cond cmds -> [cond, cmds] + T_UntilExpression _ cond cmds -> [cond, cmds] T_ForIn _ _ _ cmds -> [cmds] T_ForArithmetic _ _ _ _ cmds -> [cmds] - T_IfExpression _ thens elses -> map snd thens ++ [elses] + T_IfExpression _ thens elses -> (concatMap (\(a,b) -> [a,b]) thens) ++ [elses] T_Annotation _ _ t -> getCommandSequences t + + T_DollarExpansion _ cmds -> [cmds] + T_DollarBraceCommandExpansion _ cmds -> [cmds] + T_Backticked _ cmds -> [cmds] _ -> [] -- Get a list of names of associative arrays diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 5ad51d7..ec76910 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2297,13 +2297,30 @@ prop_checkWhileReadPitfalls5 = verifyNot checkWhileReadPitfalls "while read foo; prop_checkWhileReadPitfalls6 = verifyNot checkWhileReadPitfalls "while read foo <&3; do ssh $foo; done 3< foo" prop_checkWhileReadPitfalls7 = verify checkWhileReadPitfalls "while read foo; do if true; then ssh $foo uptime; fi; done < file" prop_checkWhileReadPitfalls8 = verifyNot checkWhileReadPitfalls "while read foo; do ssh -n $foo uptime; done < file" +prop_checkWhileReadPitfalls9 = verify checkWhileReadPitfalls "while read foo; do ffmpeg -i foo.mkv bar.mkv -an; done" +prop_checkWhileReadPitfalls10 = verify checkWhileReadPitfalls "while read foo; do mplayer foo.ogv > file; done" +prop_checkWhileReadPitfalls11 = verifyNot checkWhileReadPitfalls "while read foo; do mplayer foo.ogv <<< q; done" +prop_checkWhileReadPitfalls12 = verifyNot checkWhileReadPitfalls "while read foo\ndo\nmplayer foo.ogv << EOF\nq\nEOF\ndone" +prop_checkWhileReadPitfalls13 = verify checkWhileReadPitfalls "while read foo; do x=$(ssh host cmd); done" +prop_checkWhileReadPitfalls14 = verify checkWhileReadPitfalls "while read foo; do echo $(ssh host cmd) < /dev/null; done" -checkWhileReadPitfalls _ (T_WhileExpression id [command] contents) +checkWhileReadPitfalls params (T_WhileExpression id [command] contents) | isStdinReadCommand command = mapM_ checkMuncher contents where - munchers = [ "ssh", "ffmpeg", "mplayer", "HandBrakeCLI" ] - preventionFlags = ["n", "noconsolecontrols" ] + -- Map of munching commands to a function that checks if the flags should exclude it + munchers = Map.fromList [ + ("ssh", (hasFlag, addFlag, "-n")), + ("ffmpeg", (hasArgument, addFlag, "-nostdin")), + ("mplayer", (hasArgument, addFlag, "-noconsolecontrols")), + ("HandBrakeCLI", (\_ _ -> False, addRedirect, "< /dev/null")) + ] + -- Use flag parsing, e.g. "-an" -> "a", "n" + hasFlag ('-':flag) = elem flag . map snd . getAllFlags + -- Simple string match, e.g. "-an" -> "-an" + hasArgument arg = elem arg . mapMaybe getLiteralString . fromJust . getCommandArgv + addFlag string cmd = fixWith [replaceEnd (getId $ getCommandTokenOrThis cmd) params 0 (' ':string)] + addRedirect string cmd = fixWith [replaceEnd (getId cmd) params 0 (' ':string)] isStdinReadCommand (T_Pipeline _ _ [T_Redirecting id redirs cmd]) = let plaintext = oversimplify cmd @@ -2312,28 +2329,47 @@ checkWhileReadPitfalls _ (T_WhileExpression id [command] contents) && all (not . stdinRedirect) redirs isStdinReadCommand _ = False - checkMuncher (T_Pipeline _ _ (T_Redirecting _ redirs cmd:_)) | not $ any stdinRedirect redirs = - case cmd of - (T_IfExpression _ thens elses) -> - mapM_ checkMuncher . concat $ map fst thens ++ map snd thens ++ [elses] + checkMuncher :: Token -> Writer [TokenComment] () + checkMuncher (T_Pipeline _ _ (T_Redirecting _ redirs cmd:_)) = do + -- Check command substitutions regardless of the command + sequence_ $ do + (T_SimpleCommand _ vars args) <- Just cmd + let words = concat $ concatMap getCommandSequences $ concatMap getWords $ vars ++ args + return $ mapM_ checkMuncher words - _ -> sequence_ $ do + when (not $ any stdinRedirect redirs) $ do + -- Recurse into ifs/loops/groups/etc if this doesn't redirect + mapM_ checkMuncher $ concat $ getCommandSequences cmd + + -- Check the actual command + sequence_ $ do name <- getCommandBasename cmd - guard $ name `elem` munchers + (check, fix, flag) <- Map.lookup name munchers + guard $ not (check flag cmd) - -- Sloppily check if the command has a flag to prevent eating stdin. - let flags = getAllFlags cmd - guard . not $ any (`elem` preventionFlags) $ map snd flags return $ do info id 2095 $ name ++ " may swallow stdin, preventing this loop from working properly." - warn (getId cmd) 2095 $ - "Add < /dev/null to prevent " ++ name ++ " from swallowing stdin." + warnWithFix (getId cmd) 2095 + ("Use " ++ name ++ " " ++ flag ++ " to prevent " ++ name ++ " from swallowing stdin.") + (fix flag cmd) checkMuncher _ = return () - stdinRedirect (T_FdRedirect _ fd _) - | null fd || fd == "0" = True + stdinRedirect (T_FdRedirect _ fd op) + | fd == "0" = True + | fd == "" = + case op of + T_IoFile _ (T_Less _) _ -> True + T_IoDuplicate _ (T_LESSAND _) _ -> True + T_HereString _ _ -> True + T_HereDoc {} -> True + _ -> False stdinRedirect _ = False + + getWords t = + case t of + T_Assignment _ _ _ _ x -> getWordParts x + _ -> getWordParts t checkWhileReadPitfalls _ _ = return () From a57f6d2886e899133218e238fb1a3b26f948010d Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 15 Mar 2020 11:29:32 -0700 Subject: [PATCH 294/763] Improve detection of for loops with single values --- CHANGELOG.md | 1 + src/ShellCheck/ASTLib.hs | 1 + src/ShellCheck/Analytics.hs | 27 +++++++++++++++++++++------ 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 11860ce..0dc775a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ - SC2255: Suggest using `$((..))` in `[ 2*3 -eq 6 ]` - SC2256: Warn about translated strings that are known variables - SC2257: Warn about arithmetic mutation in redirections +- SC2258: Warn about trailing commas in for loop elements ### Changed - SC2230: 'command -v' suggestion is now off by default (-i deprecate-which) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index bc97404..2b40705 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -47,6 +47,7 @@ willSplit x = T_BraceExpansion {} -> True T_Glob {} -> True T_Extglob {} -> True + T_DoubleQuoted _ l -> any willBecomeMultipleArgs l T_NormalWord _ l -> any willSplit l _ -> False diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index ec76910..f1c36fa 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -578,16 +578,30 @@ prop_checkForInQuoted4 = verify checkForInQuoted "for f in 1,2,3; do true; done" prop_checkForInQuoted4a = verifyNot checkForInQuoted "for f in foo{1,2,3}; do true; done" prop_checkForInQuoted5 = verify checkForInQuoted "for f in ls; do true; done" prop_checkForInQuoted6 = verifyNot checkForInQuoted "for f in \"${!arr}\"; do true; done" +prop_checkForInQuoted7 = verify checkForInQuoted "for f in ls, grep, mv; do true; done" +prop_checkForInQuoted8 = verify checkForInQuoted "for f in 'ls', 'grep', 'mv'; do true; done" +prop_checkForInQuoted9 = verifyNot checkForInQuoted "for f in 'ls,' 'grep,' 'mv'; do true; done" checkForInQuoted _ (T_ForIn _ f [T_NormalWord _ [word@(T_DoubleQuoted id list)]] _) | any (\x -> willSplit x && not (mayBecomeMultipleArgs x)) list || (fmap wouldHaveBeenGlob (getLiteralString word) == Just True) = err id 2066 "Since you double quoted this, it will not word split, and the loop will only run once." checkForInQuoted _ (T_ForIn _ f [T_NormalWord _ [T_SingleQuoted id _]] _) = warn id 2041 "This is a literal string. To run as a command, use $(..) instead of '..' . " -checkForInQuoted _ (T_ForIn _ f [T_NormalWord _ [T_Literal id s]] _) = - if ',' `elem` s && '{' `notElem` s - then warn id 2042 "Use spaces, not commas, to separate loop elements." - else warn id 2043 "This loop will only ever run once for a constant value. Did you perhaps mean to loop over dir/*, $var or $(cmd)?" +checkForInQuoted _ (T_ForIn _ _ [single] _) | fromMaybe False $ fmap (',' `elem`) $ getUnquotedLiteral single = + warn (getId single) 2042 "Use spaces, not commas, to separate loop elements." +checkForInQuoted _ (T_ForIn _ _ [single] _) | not (willSplit single) && not (mayBecomeMultipleArgs single) = + warn (getId single) 2043 "This loop will only ever run once. Bad quoting or missing glob/expansion?" +checkForInQuoted params (T_ForIn _ _ multiple _) = + mapM_ f multiple + where + f arg = sequence_ $ do + suffix <- getTrailingUnquotedLiteral arg + string <- getLiteralString suffix + guard $ "," `isSuffixOf` string + return $ + warnWithFix (getId arg) 2258 + "The trailing comma is part of the value, not a separator. Delete or quote it." + (fixWith [replaceEnd (getId suffix) params 1 ""]) checkForInQuoted _ _ = return () prop_checkForInCat1 = verify checkForInCat "for f in $(cat foo); do stuff; done" @@ -1011,8 +1025,9 @@ prop_checkUnquotedN2 = verify checkUnquotedN "[ -n $cow ]" prop_checkUnquotedN3 = verifyNot checkUnquotedN "[[ -n $foo ]] && echo cow" prop_checkUnquotedN4 = verify checkUnquotedN "[ -n $cow -o -t 1 ]" prop_checkUnquotedN5 = verifyNot checkUnquotedN "[ -n \"$@\" ]" -checkUnquotedN _ (TC_Unary _ SingleBracket "-n" (T_NormalWord id [t])) | willSplit t = - err id 2070 "-n doesn't work with unquoted arguments. Quote or use [[ ]]." +checkUnquotedN _ (TC_Unary _ SingleBracket "-n" t) | willSplit t = + unless (any isArrayExpansion $ getWordParts t) $ -- There's SC2198 for these + err (getId t) 2070 "-n doesn't work with unquoted arguments. Quote or use [[ ]]." checkUnquotedN _ _ = return () prop_checkNumberComparisons1 = verify checkNumberComparisons "[[ $foo < 3 ]]" From acee69676be5f8197eb9661fda0c6aad0136502b Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 15 Mar 2020 12:54:54 -0700 Subject: [PATCH 295/763] Try to make TravisCI not fail on deployment of Docker stage --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 428fc8e..a5ff553 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,6 +21,7 @@ jobs: script: - source ./.multi_arch_docker - set -ex; multi_arch_docker::main; set +x + - mkdir deploy before_install: | DOCKER_BASE="$DOCKER_USERNAME/shellcheck" From 86d470c74fa99e79267e102034d6d507d7580d74 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 15 Mar 2020 16:05:55 -0400 Subject: [PATCH 296/763] Simplify checkForInQuoted * Avoid some unnecessary fmaps * Reuse an identical pattern-match for two guards * Apply De Morgan's law * Use forM_ to avoid an unnecessary where --- src/ShellCheck/Analytics.hs | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index f1c36fa..fc1bf2c 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -583,18 +583,17 @@ prop_checkForInQuoted8 = verify checkForInQuoted "for f in 'ls', 'grep', 'mv'; d prop_checkForInQuoted9 = verifyNot checkForInQuoted "for f in 'ls,' 'grep,' 'mv'; do true; done" checkForInQuoted _ (T_ForIn _ f [T_NormalWord _ [word@(T_DoubleQuoted id list)]] _) | any (\x -> willSplit x && not (mayBecomeMultipleArgs x)) list - || (fmap wouldHaveBeenGlob (getLiteralString word) == Just True) = + || maybe False wouldHaveBeenGlob (getLiteralString word) = err id 2066 "Since you double quoted this, it will not word split, and the loop will only run once." checkForInQuoted _ (T_ForIn _ f [T_NormalWord _ [T_SingleQuoted id _]] _) = warn id 2041 "This is a literal string. To run as a command, use $(..) instead of '..' . " -checkForInQuoted _ (T_ForIn _ _ [single] _) | fromMaybe False $ fmap (',' `elem`) $ getUnquotedLiteral single = - warn (getId single) 2042 "Use spaces, not commas, to separate loop elements." -checkForInQuoted _ (T_ForIn _ _ [single] _) | not (willSplit single) && not (mayBecomeMultipleArgs single) = - warn (getId single) 2043 "This loop will only ever run once. Bad quoting or missing glob/expansion?" +checkForInQuoted _ (T_ForIn _ _ [single] _) + | maybe False (',' `elem`) $ getUnquotedLiteral single = + warn (getId single) 2042 "Use spaces, not commas, to separate loop elements." + | not (willSplit single || mayBecomeMultipleArgs single) = + warn (getId single) 2043 "This loop will only ever run once. Bad quoting or missing glob/expansion?" checkForInQuoted params (T_ForIn _ _ multiple _) = - mapM_ f multiple - where - f arg = sequence_ $ do + forM_ multiple $ \arg -> sequence_ $ do suffix <- getTrailingUnquotedLiteral arg string <- getLiteralString suffix guard $ "," `isSuffixOf` string From 9d5363377ef11ce0aaaf9b5b66f654f1e6a475d7 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Mon, 16 Mar 2020 00:12:19 -0400 Subject: [PATCH 297/763] Simplify checkWhileReadPitfalls * Clean up usage of not * Use a case match instead of sequence_ and a do block --- src/ShellCheck/Analytics.hs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index f1c36fa..96e0750 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2341,18 +2341,18 @@ checkWhileReadPitfalls params (T_WhileExpression id [command] contents) let plaintext = oversimplify cmd in headOrDefault "" plaintext == "read" && ("-u" `notElem` plaintext) - && all (not . stdinRedirect) redirs + && not (any stdinRedirect redirs) isStdinReadCommand _ = False checkMuncher :: Token -> Writer [TokenComment] () checkMuncher (T_Pipeline _ _ (T_Redirecting _ redirs cmd:_)) = do -- Check command substitutions regardless of the command - sequence_ $ do - (T_SimpleCommand _ vars args) <- Just cmd - let words = concat $ concatMap getCommandSequences $ concatMap getWords $ vars ++ args - return $ mapM_ checkMuncher words + case cmd of + T_SimpleCommand _ vars args -> + mapM_ checkMuncher $ concat $ concatMap getCommandSequences $ concatMap getWords $ vars ++ args + _ -> return () - when (not $ any stdinRedirect redirs) $ do + unless (any stdinRedirect redirs) $ do -- Recurse into ifs/loops/groups/etc if this doesn't redirect mapM_ checkMuncher $ concat $ getCommandSequences cmd From 7a5e261d03f16e6e0be2290dc3fd3ed7d02c1650 Mon Sep 17 00:00:00 2001 From: girst Date: Mon, 16 Mar 2020 23:04:54 +0100 Subject: [PATCH 298/763] recognize `: ${parameter=word}` as assignment --- src/ShellCheck/Analytics.hs | 1 + src/ShellCheck/AnalyzerLib.hs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index f1c36fa..c4bb9c6 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2205,6 +2205,7 @@ prop_checkUnassignedReferences36= verifyNotTree checkUnassignedReferences "read prop_checkUnassignedReferences37= verifyNotTree checkUnassignedReferences "var=howdy; printf -v 'array[0]' %s \"$var\"; printf %s \"${array[0]}\";" prop_checkUnassignedReferences38= verifyTree (checkUnassignedReferences' True) "echo $VAR" prop_checkUnassignedReferences39= verifyNotTree checkUnassignedReferences "builtin export var=4; echo $var" +prop_checkUnassignedReferences40= verifyNotTree checkUnassignedReferences ": ${foo=bar}" checkUnassignedReferences = checkUnassignedReferences' False checkUnassignedReferences' includeGlobals params t = warnings diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 5f33b30..453d53d 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -508,7 +508,7 @@ getModifiedVariables t = T_DollarBraced _ _ l -> maybeToList $ do let string = bracedString t let modifier = getBracedModifier string - guard $ ":=" `isPrefixOf` modifier + guard $ any (`isPrefixOf` modifier) ["=", ":="] return (t, t, getBracedReference string, DataString $ SourceFrom [l]) t@(T_FdRedirect _ ('{':var) op) -> -- {foo}>&2 modifies foo From 7963eeab9d51b5210f21de13eaf2140805287cef Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 16 Mar 2020 21:36:41 -0700 Subject: [PATCH 299/763] Include shebang in AST traversal (fixes #1858) --- src/ShellCheck/AST.hs | 6 +++++- src/ShellCheck/Checker.hs | 7 +++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/AST.hs b/src/ShellCheck/AST.hs index d8faec6..6b13282 100644 --- a/src/ShellCheck/AST.hs +++ b/src/ShellCheck/AST.hs @@ -249,7 +249,11 @@ analyze f g i = list <- mapM round group return $ T_ForArithmetic id x y z list - delve (T_Script id s l) = dl l $ T_Script id s + delve (T_Script id shebang list) = do + newShebang <- round shebang + newList <- roundAll list + return $ T_Script id newShebang newList + delve (T_Function id a b name body) = d1 body $ T_Function id a b name delve (T_Condition id typ token) = d1 token $ T_Condition id typ delve (T_Extglob id str l) = dl l $ T_Extglob id str diff --git a/src/ShellCheck/Checker.hs b/src/ShellCheck/Checker.hs index 60280ec..120f7d8 100644 --- a/src/ShellCheck/Checker.hs +++ b/src/ShellCheck/Checker.hs @@ -288,6 +288,13 @@ prop_deducesTypeFromExtension2 = result == [2079] csScript = "(( 3.14 ))" } +prop_canDisableShebangWarning = null $ result + where + result = checkWithSpec [] emptyCheckSpec { + csFilename = "file.sh", + csScript = "#shellcheck disable=SC2148\nfoo" + } + prop_shExtensionDoesntMatter = result == [2148] where result = checkWithSpec [] emptyCheckSpec { From 9f833770b08fbfb84c201acd1275049f900d5cd4 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Thu, 19 Mar 2020 21:31:10 -0400 Subject: [PATCH 300/763] Mark that base >= 4.8.0.0 is required We've actually already required base >= 4.8.0.0 since commit a8376a0 (in which we first used `<$>` without an import, which wasn't in the Prelude prior to this version). Since then, we've also made use of other more substantial features that de-facto require base >= 4.8.0.0 since they require GHC 7.10, such as `DeriveAnyClass`. --- ShellCheck.cabal | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ShellCheck.cabal b/ShellCheck.cabal index 0389be6..a848d9c 100644 --- a/ShellCheck.cabal +++ b/ShellCheck.cabal @@ -45,9 +45,7 @@ library build-depends: aeson, array, - -- GHC 7.6.3 (base 4.6.0.1) is buggy (#1131, #1119) in optimized mode. - -- Just disable that version entirely to fail fast. - base > 4.6.0.1 && < 5, + base >= 4.8.0.0 && < 5, bytestring, containers >= 0.5, deepseq >= 1.4.0.0, From 37e78141bd57f56b2a8d7ec0c91e76edec6a9dc5 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 26 Mar 2020 17:02:08 -0700 Subject: [PATCH 301/763] Stop deploying artifacts to GCS --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a5ff553..2c76574 100644 --- a/.travis.yml +++ b/.travis.yml @@ -52,7 +52,7 @@ deploy: access_key_id: GOOG7MDN7WEH6IIGBDCA secret_access_key: secure: Bcx2cT0/E2ikj7sdamVq52xlLZF9dz9ojGPtoKfPyQhkkZa+McVI4xgUSuyyoSxyKj77sofx2y8m6PJYYumT4g5hREV1tfeUkl0J2DQFMbGDYEt7kxVkXCxojNvhHwTzLFv0ezstrxWWxQm81BfQQ4U9lggRXtndAP4czZnOeHPINPSiue1QNwRAEw05r5UoIUJXy/5xyUrjIxn381pAs+gJqP2COeN9kTKYH53nS/AAws29RprfZFnPlo7xxWmcjRcdS5KPdGXI/c6tQp5zl2iTh510VC1PN2w1Wvnn/oNWhiNdqPyVDsojIX5+sS3nejzJA+KFMxXSBlyXIY3wPpS/MdscU79X6Q5f9ivsFfsm7gNBmxHUPNn0HAvU4ROT/CCE9j6jSbs5PC7QBo3CK4++jxAwE/pd9HUc2rs3k0ofx3rgveJ7txpy5yPKfwIIBi98kVKlC4w7dLvNTOfjW1Imt2yH87XTfsE0UIG9st1WII6s4l/WgBx2GuwKdt6+3QUYiAlCFckkxWi+fAvpHZUEL43Qxub5fN+ZV7Zib1n7opchH4QKGBb6/y0WaDCmtCfu0lppoe/TH6saOTjDFj67NJSElK6ZDxGZ3uw4R+ret2gm6WRKT2Oeub8J33VzSa7VkmFpMPrAAfPa9N1Z4ewBLoTmvxSg2A0dDrCdJio= - bucket: shellcheck + bucket: shellcheck-private local_dir: deploy on: repo: koalaman/shellcheck From 615063a9c3ace19c81a8529762c0601b331bb2a2 Mon Sep 17 00:00:00 2001 From: Artur Klauser Date: Sun, 22 Mar 2020 09:01:21 +0100 Subject: [PATCH 302/763] Don't try to deploy docker images on PR runs For security reasons, PR runs don't have access to Travis secrets. However, Docker deployment depends on the secret DOCKER_PASSWORD. Thus we shouldn't try Docker deployment when running PRs since it will fail for lack of access. --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 2c76574..5207655 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,8 @@ jobs: os: osx - stage: Deploy docker image - if: branch = master + # Deploy only for pushes to master branch, not other branches, not PRs. + if: branch = master AND type = push script: - source ./.multi_arch_docker - set -ex; multi_arch_docker::main; set +x From 8cf037fe5e78d6894051d4f070d7f0343130e521 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sat, 28 Mar 2020 18:29:22 -0400 Subject: [PATCH 303/763] Fix #1892: Use pattern synonyms to clean up AST --- src/ShellCheck/AST.hs | 545 +++++++++++++++++------------------------- 1 file changed, 214 insertions(+), 331 deletions(-) diff --git a/src/ShellCheck/AST.hs b/src/ShellCheck/AST.hs index 6b13282..d4151fe 100644 --- a/src/ShellCheck/AST.hs +++ b/src/ShellCheck/AST.hs @@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . -} -{-# LANGUAGE DeriveGeneric, DeriveAnyClass #-} +{-# LANGUAGE DeriveGeneric, DeriveAnyClass, DeriveTraversable, PatternSynonyms #-} module ShellCheck.AST where import GHC.Generics (Generic) @@ -37,110 +37,112 @@ newtype FunctionParentheses = FunctionParentheses Bool deriving (Show, Eq) data CaseType = CaseBreak | CaseFallThrough | CaseContinue deriving (Show, Eq) newtype Root = Root Token -data Token = - TA_Binary Id String Token Token - | TA_Assignment Id String Token Token - | TA_Variable Id String [Token] - | TA_Expansion Id [Token] - | TA_Sequence Id [Token] - | TA_Trinary Id Token Token Token - | TA_Unary Id String Token - | TC_And Id ConditionType String Token Token - | TC_Binary Id ConditionType String Token Token - | TC_Group Id ConditionType Token - | TC_Nullary Id ConditionType Token - | TC_Or Id ConditionType String Token Token - | TC_Unary Id ConditionType String Token - | TC_Empty Id ConditionType - | T_AND_IF Id - | T_AndIf Id Token Token - | T_Arithmetic Id Token - | T_Array Id [Token] - | T_IndexedElement Id [Token] Token +data Token = OuterToken Id (InnerToken Token) deriving (Show) + +data InnerToken t = + Inner_TA_Binary String t t + | Inner_TA_Assignment String t t + | Inner_TA_Variable String [t] + | Inner_TA_Expansion [t] + | Inner_TA_Sequence [t] + | Inner_TA_Trinary t t t + | Inner_TA_Unary String t + | Inner_TC_And ConditionType String t t + | Inner_TC_Binary ConditionType String t t + | Inner_TC_Group ConditionType t + | Inner_TC_Nullary ConditionType t + | Inner_TC_Or ConditionType String t t + | Inner_TC_Unary ConditionType String t + | Inner_TC_Empty ConditionType + | Inner_T_AND_IF + | Inner_T_AndIf t t + | Inner_T_Arithmetic t + | Inner_T_Array [t] + | Inner_T_IndexedElement [t] t -- Store the index as string, and parse as arithmetic or string later - | T_UnparsedIndex Id SourcePos String - | T_Assignment Id AssignmentMode String [Token] Token - | T_Backgrounded Id Token - | T_Backticked Id [Token] - | T_Bang Id - | T_Banged Id Token - | T_BraceExpansion Id [Token] - | T_BraceGroup Id [Token] - | T_CLOBBER Id - | T_Case Id - | T_CaseExpression Id Token [(CaseType, [Token], [Token])] - | T_Condition Id ConditionType Token - | T_DGREAT Id - | T_DLESS Id - | T_DLESSDASH Id - | T_DSEMI Id - | T_Do Id - | T_DollarArithmetic Id Token - | T_DollarBraced Id Bool Token - | T_DollarBracket Id Token - | T_DollarDoubleQuoted Id [Token] - | T_DollarExpansion Id [Token] - | T_DollarSingleQuoted Id String - | T_DollarBraceCommandExpansion Id [Token] - | T_Done Id - | T_DoubleQuoted Id [Token] - | T_EOF Id - | T_Elif Id - | T_Else Id - | T_Esac Id - | T_Extglob Id String [Token] - | T_FdRedirect Id String Token - | T_Fi Id - | T_For Id - | T_ForArithmetic Id Token Token Token [Token] - | T_ForIn Id String [Token] [Token] - | T_Function Id FunctionKeyword FunctionParentheses String Token - | T_GREATAND Id - | T_Glob Id String - | T_Greater Id - | T_HereDoc Id Dashed Quoted String [Token] - | T_HereString Id Token - | T_If Id - | T_IfExpression Id [([Token],[Token])] [Token] - | T_In Id - | T_IoFile Id Token Token - | T_IoDuplicate Id Token String - | T_LESSAND Id - | T_LESSGREAT Id - | T_Lbrace Id - | T_Less Id - | T_Literal Id String - | T_Lparen Id - | T_NEWLINE Id - | T_NormalWord Id [Token] - | T_OR_IF Id - | T_OrIf Id Token Token - | T_ParamSubSpecialChar Id String -- e.g. '%' in ${foo%bar} or '/' in ${foo/bar/baz} - | T_Pipeline Id [Token] [Token] -- [Pipe separators] [Commands] - | T_ProcSub Id String [Token] - | T_Rbrace Id - | T_Redirecting Id [Token] Token - | T_Rparen Id - | T_Script Id Token [Token] -- Shebang T_Literal, followed by script. - | T_Select Id - | T_SelectIn Id String [Token] [Token] - | T_Semi Id - | T_SimpleCommand Id [Token] [Token] - | T_SingleQuoted Id String - | T_Subshell Id [Token] - | T_Then Id - | T_Until Id - | T_UntilExpression Id [Token] [Token] - | T_While Id - | T_WhileExpression Id [Token] [Token] - | T_Annotation Id [Annotation] Token - | T_Pipe Id String - | T_CoProc Id (Maybe String) Token - | T_CoProcBody Id Token - | T_Include Id Token - | T_SourceCommand Id Token Token - | T_BatsTest Id Token Token - deriving (Show) + | Inner_T_UnparsedIndex SourcePos String + | Inner_T_Assignment AssignmentMode String [t] t + | Inner_T_Backgrounded t + | Inner_T_Backticked [t] + | Inner_T_Bang + | Inner_T_Banged t + | Inner_T_BraceExpansion [t] + | Inner_T_BraceGroup [t] + | Inner_T_CLOBBER + | Inner_T_Case + | Inner_T_CaseExpression t [(CaseType, [t], [t])] + | Inner_T_Condition ConditionType t + | Inner_T_DGREAT + | Inner_T_DLESS + | Inner_T_DLESSDASH + | Inner_T_DSEMI + | Inner_T_Do + | Inner_T_DollarArithmetic t + | Inner_T_DollarBraced Bool t + | Inner_T_DollarBracket t + | Inner_T_DollarDoubleQuoted [t] + | Inner_T_DollarExpansion [t] + | Inner_T_DollarSingleQuoted String + | Inner_T_DollarBraceCommandExpansion [t] + | Inner_T_Done + | Inner_T_DoubleQuoted [t] + | Inner_T_EOF + | Inner_T_Elif + | Inner_T_Else + | Inner_T_Esac + | Inner_T_Extglob String [t] + | Inner_T_FdRedirect String t + | Inner_T_Fi + | Inner_T_For + | Inner_T_ForArithmetic t t t [t] + | Inner_T_ForIn String [t] [t] + | Inner_T_Function FunctionKeyword FunctionParentheses String t + | Inner_T_GREATAND + | Inner_T_Glob String + | Inner_T_Greater + | Inner_T_HereDoc Dashed Quoted String [t] + | Inner_T_HereString t + | Inner_T_If + | Inner_T_IfExpression [([t],[t])] [t] + | Inner_T_In + | Inner_T_IoFile t t + | Inner_T_IoDuplicate t String + | Inner_T_LESSAND + | Inner_T_LESSGREAT + | Inner_T_Lbrace + | Inner_T_Less + | Inner_T_Literal String + | Inner_T_Lparen + | Inner_T_NEWLINE + | Inner_T_NormalWord [t] + | Inner_T_OR_IF + | Inner_T_OrIf t t + | Inner_T_ParamSubSpecialChar String -- e.g. '%' in ${foo%bar} or '/' in ${foo/bar/baz} + | Inner_T_Pipeline [t] [t] -- [Pipe separators] [Commands] + | Inner_T_ProcSub String [t] + | Inner_T_Rbrace + | Inner_T_Redirecting [t] t + | Inner_T_Rparen + | Inner_T_Script t [t] -- Shebang T_Literal, followed by script. + | Inner_T_Select + | Inner_T_SelectIn String [t] [t] + | Inner_T_Semi + | Inner_T_SimpleCommand [t] [t] + | Inner_T_SingleQuoted String + | Inner_T_Subshell [t] + | Inner_T_Then + | Inner_T_Until + | Inner_T_UntilExpression [t] [t] + | Inner_T_While + | Inner_T_WhileExpression [t] [t] + | Inner_T_Annotation [Annotation] t + | Inner_T_Pipe String + | Inner_T_CoProc (Maybe String) t + | Inner_T_CoProcBody t + | Inner_T_Include t + | Inner_T_SourceCommand t t + | Inner_T_BatsTest t t + deriving (Show, Eq, Functor, Foldable, Traversable) data Annotation = DisableComment Integer @@ -151,244 +153,125 @@ data Annotation = deriving (Show, Eq) data ConditionType = DoubleBracket | SingleBracket deriving (Show, Eq) --- This is an abomination. -tokenEquals :: Token -> Token -> Bool -tokenEquals a b = kludge a == kludge b - where kludge s = Re.subRegex (Re.mkRegex "\\(Id [0-9]+\\)") (show s) "(Id 0)" +pattern T_AND_IF id = OuterToken id Inner_T_AND_IF +pattern T_Bang id = OuterToken id Inner_T_Bang +pattern T_Case id = OuterToken id Inner_T_Case +pattern TC_Empty id typ = OuterToken id (Inner_TC_Empty typ) +pattern T_CLOBBER id = OuterToken id Inner_T_CLOBBER +pattern T_DGREAT id = OuterToken id Inner_T_DGREAT +pattern T_DLESS id = OuterToken id Inner_T_DLESS +pattern T_DLESSDASH id = OuterToken id Inner_T_DLESSDASH +pattern T_Do id = OuterToken id Inner_T_Do +pattern T_DollarSingleQuoted id str = OuterToken id (Inner_T_DollarSingleQuoted str) +pattern T_Done id = OuterToken id Inner_T_Done +pattern T_DSEMI id = OuterToken id Inner_T_DSEMI +pattern T_Elif id = OuterToken id Inner_T_Elif +pattern T_Else id = OuterToken id Inner_T_Else +pattern T_EOF id = OuterToken id Inner_T_EOF +pattern T_Esac id = OuterToken id Inner_T_Esac +pattern T_Fi id = OuterToken id Inner_T_Fi +pattern T_For id = OuterToken id Inner_T_For +pattern T_Glob id str = OuterToken id (Inner_T_Glob str) +pattern T_GREATAND id = OuterToken id Inner_T_GREATAND +pattern T_Greater id = OuterToken id Inner_T_Greater +pattern T_If id = OuterToken id Inner_T_If +pattern T_In id = OuterToken id Inner_T_In +pattern T_Lbrace id = OuterToken id Inner_T_Lbrace +pattern T_Less id = OuterToken id Inner_T_Less +pattern T_LESSAND id = OuterToken id Inner_T_LESSAND +pattern T_LESSGREAT id = OuterToken id Inner_T_LESSGREAT +pattern T_Literal id str = OuterToken id (Inner_T_Literal str) +pattern T_Lparen id = OuterToken id Inner_T_Lparen +pattern T_NEWLINE id = OuterToken id Inner_T_NEWLINE +pattern T_OR_IF id = OuterToken id Inner_T_OR_IF +pattern T_ParamSubSpecialChar id str = OuterToken id (Inner_T_ParamSubSpecialChar str) +pattern T_Pipe id str = OuterToken id (Inner_T_Pipe str) +pattern T_Rbrace id = OuterToken id Inner_T_Rbrace +pattern T_Rparen id = OuterToken id Inner_T_Rparen +pattern T_Select id = OuterToken id Inner_T_Select +pattern T_Semi id = OuterToken id Inner_T_Semi +pattern T_SingleQuoted id str = OuterToken id (Inner_T_SingleQuoted str) +pattern T_Then id = OuterToken id Inner_T_Then +pattern T_UnparsedIndex id pos str = OuterToken id (Inner_T_UnparsedIndex pos str) +pattern T_Until id = OuterToken id Inner_T_Until +pattern T_While id = OuterToken id Inner_T_While +pattern TA_Assignment id op t1 t2 = OuterToken id (Inner_TA_Assignment op t1 t2) +pattern TA_Binary id op t1 t2 = OuterToken id (Inner_TA_Binary op t1 t2) +pattern TA_Expansion id t = OuterToken id (Inner_TA_Expansion t) +pattern T_AndIf id t u = OuterToken id (Inner_T_AndIf t u) +pattern T_Annotation id anns t = OuterToken id (Inner_T_Annotation anns t) +pattern T_Arithmetic id c = OuterToken id (Inner_T_Arithmetic c) +pattern T_Array id t = OuterToken id (Inner_T_Array t) +pattern TA_Sequence id l = OuterToken id (Inner_TA_Sequence l) +pattern T_Assignment id mode var indices value = OuterToken id (Inner_T_Assignment mode var indices value) +pattern TA_Trinary id t1 t2 t3 = OuterToken id (Inner_TA_Trinary t1 t2 t3) +pattern TA_Unary id op t1 = OuterToken id (Inner_TA_Unary op t1) +pattern TA_Variable id str t = OuterToken id (Inner_TA_Variable str t) +pattern T_Backgrounded id l = OuterToken id (Inner_T_Backgrounded l) +pattern T_Backticked id list = OuterToken id (Inner_T_Backticked list) +pattern T_Banged id l = OuterToken id (Inner_T_Banged l) +pattern T_BatsTest id name t = OuterToken id (Inner_T_BatsTest name t) +pattern T_BraceExpansion id list = OuterToken id (Inner_T_BraceExpansion list) +pattern T_BraceGroup id l = OuterToken id (Inner_T_BraceGroup l) +pattern TC_And id typ str t1 t2 = OuterToken id (Inner_TC_And typ str t1 t2) +pattern T_CaseExpression id word cases = OuterToken id (Inner_T_CaseExpression word cases) +pattern TC_Binary id typ op lhs rhs = OuterToken id (Inner_TC_Binary typ op lhs rhs) +pattern TC_Group id typ token = OuterToken id (Inner_TC_Group typ token) +pattern TC_Nullary id typ token = OuterToken id (Inner_TC_Nullary typ token) +pattern T_Condition id typ token = OuterToken id (Inner_T_Condition typ token) +pattern T_CoProcBody id t = OuterToken id (Inner_T_CoProcBody t) +pattern T_CoProc id var body = OuterToken id (Inner_T_CoProc var body) +pattern TC_Or id typ str t1 t2 = OuterToken id (Inner_TC_Or typ str t1 t2) +pattern TC_Unary id typ op token = OuterToken id (Inner_TC_Unary typ op token) +pattern T_DollarArithmetic id c = OuterToken id (Inner_T_DollarArithmetic c) +pattern T_DollarBraceCommandExpansion id list = OuterToken id (Inner_T_DollarBraceCommandExpansion list) +pattern T_DollarBraced id braced op = OuterToken id (Inner_T_DollarBraced braced op) +pattern T_DollarBracket id c = OuterToken id (Inner_T_DollarBracket c) +pattern T_DollarDoubleQuoted id list = OuterToken id (Inner_T_DollarDoubleQuoted list) +pattern T_DollarExpansion id list = OuterToken id (Inner_T_DollarExpansion list) +pattern T_DoubleQuoted id list = OuterToken id (Inner_T_DoubleQuoted list) +pattern T_Extglob id str l = OuterToken id (Inner_T_Extglob str l) +pattern T_FdRedirect id v t = OuterToken id (Inner_T_FdRedirect v t) +pattern T_ForArithmetic id a b c group = OuterToken id (Inner_T_ForArithmetic a b c group) +pattern T_ForIn id v w l = OuterToken id (Inner_T_ForIn v w l) +pattern T_Function id a b name body = OuterToken id (Inner_T_Function a b name body) +pattern T_HereDoc id d q str l = OuterToken id (Inner_T_HereDoc d q str l) +pattern T_HereString id word = OuterToken id (Inner_T_HereString word) +pattern T_IfExpression id conditions elses = OuterToken id (Inner_T_IfExpression conditions elses) +pattern T_Include id script = OuterToken id (Inner_T_Include script) +pattern T_IndexedElement id indices t = OuterToken id (Inner_T_IndexedElement indices t) +pattern T_IoDuplicate id op num = OuterToken id (Inner_T_IoDuplicate op num) +pattern T_IoFile id op file = OuterToken id (Inner_T_IoFile op file) +pattern T_NormalWord id list = OuterToken id (Inner_T_NormalWord list) +pattern T_OrIf id t u = OuterToken id (Inner_T_OrIf t u) +pattern T_Pipeline id l1 l2 = OuterToken id (Inner_T_Pipeline l1 l2) +pattern T_ProcSub id typ l = OuterToken id (Inner_T_ProcSub typ l) +pattern T_Redirecting id redirs cmd = OuterToken id (Inner_T_Redirecting redirs cmd) +pattern T_Script id shebang list = OuterToken id (Inner_T_Script shebang list) +pattern T_SelectIn id v w l = OuterToken id (Inner_T_SelectIn v w l) +pattern T_SimpleCommand id vars cmds = OuterToken id (Inner_T_SimpleCommand vars cmds) +pattern T_SourceCommand id includer t_include = OuterToken id (Inner_T_SourceCommand includer t_include) +pattern T_Subshell id l = OuterToken id (Inner_T_Subshell l) +pattern T_UntilExpression id c l = OuterToken id (Inner_T_UntilExpression c l) +pattern T_WhileExpression id c l = OuterToken id (Inner_T_WhileExpression c l) + +{-# COMPLETE T_AND_IF, T_Bang, T_Case, TC_Empty, T_CLOBBER, T_DGREAT, T_DLESS, T_DLESSDASH, T_Do, T_DollarSingleQuoted, T_Done, T_DSEMI, T_Elif, T_Else, T_EOF, T_Esac, T_Fi, T_For, T_Glob, T_GREATAND, T_Greater, T_If, T_In, T_Lbrace, T_Less, T_LESSAND, T_LESSGREAT, T_Literal, T_Lparen, T_NEWLINE, T_OR_IF, T_ParamSubSpecialChar, T_Pipe, T_Rbrace, T_Rparen, T_Select, T_Semi, T_SingleQuoted, T_Then, T_UnparsedIndex, T_Until, T_While, TA_Assignment, TA_Binary, TA_Expansion, T_AndIf, T_Annotation, T_Arithmetic, T_Array, TA_Sequence, T_Assignment, TA_Trinary, TA_Unary, TA_Variable, T_Backgrounded, T_Backticked, T_Banged, T_BatsTest, T_BraceExpansion, T_BraceGroup, TC_And, T_CaseExpression, TC_Binary, TC_Group, TC_Nullary, T_Condition, T_CoProcBody, T_CoProc, TC_Or, TC_Unary, T_DollarArithmetic, T_DollarBraceCommandExpansion, T_DollarBraced, T_DollarBracket, T_DollarDoubleQuoted, T_DollarExpansion, T_DoubleQuoted, T_Extglob, T_FdRedirect, T_ForArithmetic, T_ForIn, T_Function, T_HereDoc, T_HereString, T_IfExpression, T_Include, T_IndexedElement, T_IoDuplicate, T_IoFile, T_NormalWord, T_OrIf, T_Pipeline, T_ProcSub, T_Redirecting, T_Script, T_SelectIn, T_SimpleCommand, T_SourceCommand, T_Subshell, T_UntilExpression, T_WhileExpression #-} instance Eq Token where - (==) = tokenEquals + OuterToken _ a == OuterToken _ b = a == b analyze :: Monad m => (Token -> m ()) -> (Token -> m ()) -> (Token -> m Token) -> Token -> m Token analyze f g i = round where - round t = do + round t@(OuterToken id it) = do f t - newT <- delve t + newIt <- traverse round it g t - i newT - roundAll = mapM round - - dl l v = do - x <- roundAll l - return $ v x - dll l m v = do - x <- roundAll l - y <- roundAll m - return $ v x y - d1 t v = do - x <- round t - return $ v x - d2 t1 t2 v = do - x <- round t1 - y <- round t2 - return $ v x y - - delve (T_NormalWord id list) = dl list $ T_NormalWord id - delve (T_DoubleQuoted id list) = dl list $ T_DoubleQuoted id - delve (T_DollarDoubleQuoted id list) = dl list $ T_DollarDoubleQuoted id - delve (T_DollarExpansion id list) = dl list $ T_DollarExpansion id - delve (T_DollarBraceCommandExpansion id list) = dl list $ T_DollarBraceCommandExpansion id - delve (T_BraceExpansion id list) = dl list $ T_BraceExpansion id - delve (T_Backticked id list) = dl list $ T_Backticked id - delve (T_DollarArithmetic id c) = d1 c $ T_DollarArithmetic id - delve (T_DollarBracket id c) = d1 c $ T_DollarBracket id - delve (T_IoFile id op file) = d2 op file $ T_IoFile id - delve (T_IoDuplicate id op num) = d1 op $ \x -> T_IoDuplicate id x num - delve (T_HereString id word) = d1 word $ T_HereString id - delve (T_FdRedirect id v t) = d1 t $ T_FdRedirect id v - delve (T_Assignment id mode var indices value) = do - a <- roundAll indices - b <- round value - return $ T_Assignment id mode var a b - delve (T_Array id t) = dl t $ T_Array id - delve (T_IndexedElement id indices t) = do - a <- roundAll indices - b <- round t - return $ T_IndexedElement id a b - delve (T_Redirecting id redirs cmd) = do - a <- roundAll redirs - b <- round cmd - return $ T_Redirecting id a b - delve (T_SimpleCommand id vars cmds) = dll vars cmds $ T_SimpleCommand id - delve (T_Pipeline id l1 l2) = dll l1 l2 $ T_Pipeline id - delve (T_Banged id l) = d1 l $ T_Banged id - delve (T_AndIf id t u) = d2 t u $ T_AndIf id - delve (T_OrIf id t u) = d2 t u $ T_OrIf id - delve (T_Backgrounded id l) = d1 l $ T_Backgrounded id - delve (T_Subshell id l) = dl l $ T_Subshell id - delve (T_ProcSub id typ l) = dl l $ T_ProcSub id typ - delve (T_Arithmetic id c) = d1 c $ T_Arithmetic id - delve (T_IfExpression id conditions elses) = do - newConds <- mapM (\(c, t) -> do - x <- mapM round c - y <- mapM round t - return (x,y) - ) conditions - newElses <- roundAll elses - return $ T_IfExpression id newConds newElses - delve (T_BraceGroup id l) = dl l $ T_BraceGroup id - delve (T_WhileExpression id c l) = dll c l $ T_WhileExpression id - delve (T_UntilExpression id c l) = dll c l $ T_UntilExpression id - delve (T_ForIn id v w l) = dll w l $ T_ForIn id v - delve (T_SelectIn id v w l) = dll w l $ T_SelectIn id v - delve (T_CaseExpression id word cases) = do - newWord <- round word - newCases <- mapM (\(o, c, t) -> do - x <- mapM round c - y <- mapM round t - return (o, x,y) - ) cases - return $ T_CaseExpression id newWord newCases - - delve (T_ForArithmetic id a b c group) = do - x <- round a - y <- round b - z <- round c - list <- mapM round group - return $ T_ForArithmetic id x y z list - - delve (T_Script id shebang list) = do - newShebang <- round shebang - newList <- roundAll list - return $ T_Script id newShebang newList - - delve (T_Function id a b name body) = d1 body $ T_Function id a b name - delve (T_Condition id typ token) = d1 token $ T_Condition id typ - delve (T_Extglob id str l) = dl l $ T_Extglob id str - delve (T_DollarBraced id braced op) = d1 op $ T_DollarBraced id braced - delve (T_HereDoc id d q str l) = dl l $ T_HereDoc id d q str - - delve (TC_And id typ str t1 t2) = d2 t1 t2 $ TC_And id typ str - delve (TC_Or id typ str t1 t2) = d2 t1 t2 $ TC_Or id typ str - delve (TC_Group id typ token) = d1 token $ TC_Group id typ - delve (TC_Binary id typ op lhs rhs) = d2 lhs rhs $ TC_Binary id typ op - delve (TC_Unary id typ op token) = d1 token $ TC_Unary id typ op - delve (TC_Nullary id typ token) = d1 token $ TC_Nullary id typ - - delve (TA_Binary id op t1 t2) = d2 t1 t2 $ TA_Binary id op - delve (TA_Assignment id op t1 t2) = d2 t1 t2 $ TA_Assignment id op - delve (TA_Unary id op t1) = d1 t1 $ TA_Unary id op - delve (TA_Sequence id l) = dl l $ TA_Sequence id - delve (TA_Trinary id t1 t2 t3) = do - a <- round t1 - b <- round t2 - c <- round t3 - return $ TA_Trinary id a b c - delve (TA_Expansion id t) = dl t $ TA_Expansion id - delve (TA_Variable id str t) = dl t $ TA_Variable id str - delve (T_Annotation id anns t) = d1 t $ T_Annotation id anns - delve (T_CoProc id var body) = d1 body $ T_CoProc id var - delve (T_CoProcBody id t) = d1 t $ T_CoProcBody id - delve (T_Include id script) = d1 script $ T_Include id - delve (T_SourceCommand id includer t_include) = d2 includer t_include $ T_SourceCommand id - delve (T_BatsTest id name t) = d2 name t $ T_BatsTest id - delve t = return t + i (OuterToken id newIt) getId :: Token -> Id -getId t = case t of - T_AND_IF id -> id - T_OR_IF id -> id - T_DSEMI id -> id - T_Semi id -> id - T_DLESS id -> id - T_DGREAT id -> id - T_LESSAND id -> id - T_GREATAND id -> id - T_LESSGREAT id -> id - T_DLESSDASH id -> id - T_CLOBBER id -> id - T_If id -> id - T_Then id -> id - T_Else id -> id - T_Elif id -> id - T_Fi id -> id - T_Do id -> id - T_Done id -> id - T_Case id -> id - T_Esac id -> id - T_While id -> id - T_Until id -> id - T_For id -> id - T_Select id -> id - T_Lbrace id -> id - T_Rbrace id -> id - T_Lparen id -> id - T_Rparen id -> id - T_Bang id -> id - T_In id -> id - T_NEWLINE id -> id - T_EOF id -> id - T_Less id -> id - T_Greater id -> id - T_SingleQuoted id _ -> id - T_Literal id _ -> id - T_NormalWord id _ -> id - T_DoubleQuoted id _ -> id - T_DollarExpansion id _ -> id - T_DollarBraced id _ _ -> id - T_DollarArithmetic id _ -> id - T_BraceExpansion id _ -> id - T_ParamSubSpecialChar id _ -> id - T_DollarBraceCommandExpansion id _ -> id - T_IoFile id _ _ -> id - T_IoDuplicate id _ _ -> id - T_HereDoc id _ _ _ _ -> id - T_HereString id _ -> id - T_FdRedirect id _ _ -> id - T_Assignment id _ _ _ _ -> id - T_Array id _ -> id - T_IndexedElement id _ _ -> id - T_Redirecting id _ _ -> id - T_SimpleCommand id _ _ -> id - T_Pipeline id _ _ -> id - T_Banged id _ -> id - T_AndIf id _ _ -> id - T_OrIf id _ _ -> id - T_Backgrounded id _ -> id - T_IfExpression id _ _ -> id - T_Subshell id _ -> id - T_BraceGroup id _ -> id - T_WhileExpression id _ _ -> id - T_UntilExpression id _ _ -> id - T_ForIn id _ _ _ -> id - T_SelectIn id _ _ _ -> id - T_CaseExpression id _ _ -> id - T_Function id _ _ _ _ -> id - T_Arithmetic id _ -> id - T_Script id _ _ -> id - T_Condition id _ _ -> id - T_Extglob id _ _ -> id - T_Backticked id _ -> id - TC_And id _ _ _ _ -> id - TC_Or id _ _ _ _ -> id - TC_Group id _ _ -> id - TC_Binary id _ _ _ _ -> id - TC_Unary id _ _ _ -> id - TC_Nullary id _ _ -> id - TA_Binary id _ _ _ -> id - TA_Assignment id _ _ _ -> id - TA_Unary id _ _ -> id - TA_Sequence id _ -> id - TA_Trinary id _ _ _ -> id - TA_Expansion id _ -> id - T_ProcSub id _ _ -> id - T_Glob id _ -> id - T_ForArithmetic id _ _ _ _ -> id - T_DollarSingleQuoted id _ -> id - T_DollarDoubleQuoted id _ -> id - T_DollarBracket id _ -> id - T_Annotation id _ _ -> id - T_Pipe id _ -> id - T_CoProc id _ _ -> id - T_CoProcBody id _ -> id - T_Include id _ -> id - T_SourceCommand id _ _ -> id - T_UnparsedIndex id _ _ -> id - TC_Empty id _ -> id - TA_Variable id _ _ -> id - T_BatsTest id _ _ -> id +getId (OuterToken id _) = id blank :: Monad m => Token -> m () blank = const $ return () From 67f0dc4fd54047cca8f24231abb857a5a8ae2f48 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 30 Mar 2020 17:55:07 -0700 Subject: [PATCH 304/763] Update distro tests to support newer Cabal --- test/buildtest | 23 ++++++++++++++++++----- test/distrotest | 22 +++++++++------------- 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/test/buildtest b/test/buildtest index e3aa1eb..68bd048 100755 --- a/test/buildtest +++ b/test/buildtest @@ -11,21 +11,34 @@ command -v cabal || cabal update || die "can't update" -cabal install --dependencies-only --enable-tests || - die "can't install dependencies" -cabal configure --enable-tests || + +if [ -e /etc/arch-release ] +then + # Arch has an unconventional packaging setup + flags=(--disable-library-vanilla --enable-shared --enable-executable-dynamic --ghc-options=-dynamic) +else + flags=() +fi + +cabal install --dependencies-only --enable-tests "${flags[@]}" || + cabal install --dependencies-only "${flags[@]}" || + die "can't install dependencies" +cabal configure --enable-tests "${flags[@]}" || die "configure failed" cabal build || die "build failed" cabal test || die "test failed" -dist/build/shellcheck/shellcheck - << 'EOF' || die "execution failed" +sc="$(find . -name shellcheck -type f -perm -111)" +[ -x "$sc" ] || die "Can't find executable" + +"$sc" - << 'EOF' || die "execution failed" #!/bin/sh echo "Hello World" EOF -dist/build/shellcheck/shellcheck - << 'EOF' && die "negative execution failed" +"$sc" - << 'EOF' && die "negative execution failed" #!/bin/sh echo $1 EOF diff --git a/test/distrotest b/test/distrotest index d706185..346a706 100755 --- a/test/distrotest +++ b/test/distrotest @@ -17,13 +17,13 @@ and is still highly experimental. Make sure you're plugged in and have screen/tmux in place, then re-run with $0 --run to continue. -Also note that 'dist' will be deleted. +Also note that dist* will be deleted. EOF exit 0 } -echo "Deleting 'dist'..." -rm -rf dist +echo "Deleting 'dist' and 'dist-newstyle'..." +rm -rf dist dist-newstyle log=$(mktemp) || die "Can't create temp file" date >> "$log" || die "Can't write to log" @@ -61,20 +61,16 @@ done << EOF debian:stable apt-get update && apt-get install -y cabal-install debian:testing apt-get update && apt-get install -y cabal-install ubuntu:latest apt-get update && apt-get install -y cabal-install +haskell:latest true opensuse/leap:latest zypper install -y cabal-install ghc +fedora:latest dnf install -y cabal-install ghc-template-haskell-devel findutils +archlinux/base:latest pacman -S -y --noconfirm cabal-install ghc-static base-devel -# Other Ubuntu versions we want to support -ubuntu:19.04 apt-get update && apt-get install -y cabal-install -ubuntu:18.10 apt-get update && apt-get install -y cabal-install +# Other versions we want to support +ubuntu:18.04 apt-get update && apt-get install -y cabal-install # Misc Haskell including current and latest Stack build -ubuntu:18.10 set -e; apt-get update && apt-get install -y curl && curl -sSL https://get.haskellstack.org/ | sh -s - -f && cd /mnt && exec test/stacktest -haskell:latest true - -# Known to currently fail -centos:latest yum install -y epel-release && yum install -y cabal-install -fedora:latest dnf install -y cabal-install -archlinux/base:latest pacman -S -y --noconfirm cabal-install ghc-static base-devel +ubuntu:18.04 set -e; apt-get update && apt-get install -y curl && curl -sSL https://get.haskellstack.org/ | sh -s - -f && cd /mnt && exec test/stacktest EOF exit "$final" From a128796c0c1244ce228aee39842d64da25c82718 Mon Sep 17 00:00:00 2001 From: Artur Klauser Date: Sun, 22 Mar 2020 08:55:11 +0100 Subject: [PATCH 305/763] Run "deploy" step only for "Build" stages --- .travis.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5207655..e2dc0bc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,8 +22,8 @@ jobs: script: - source ./.multi_arch_docker - set -ex; multi_arch_docker::main; set +x - - mkdir deploy +# This is in global context and runs for every stage that doesn't override it. before_install: | DOCKER_BASE="$DOCKER_USERNAME/shellcheck" DOCKER_BUILDS="" @@ -32,6 +32,7 @@ before_install: | test -n "$TRAVIS_TAG" && TAGS="$TAGS stable $TRAVIS_TAG" || true echo "Tags are $TAGS" +# This is in global context and runs for every stage that doesn't override it. script: - mkdir -p deploy - source ./.compile_binaries @@ -40,6 +41,7 @@ script: - ./.prepare_deploy - ./.github_deploy +# This is in global context and runs for every stage that doesn't override it. after_failure: | id pwd @@ -47,6 +49,7 @@ after_failure: | find . -name '*.log' -type f -exec grep "" /dev/null {} + find . -ls +# This is in global context and runs for every stage that doesn't override it. deploy: provider: gcs skip_cleanup: true @@ -57,4 +60,5 @@ deploy: local_dir: deploy on: repo: koalaman/shellcheck + condition: $TRAVIS_BUILD_STAGE_NAME = Build all_branches: true From 93782275701ef9f504d8c1c4414d2822673b8af8 Mon Sep 17 00:00:00 2001 From: Artur Klauser Date: Wed, 1 Apr 2020 09:03:38 +0200 Subject: [PATCH 306/763] Use shellcheck on yourself Fixing shellcheck warnings on shell scripts in this repo. --- nextnumber | 2 +- test/check_release | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/nextnumber b/nextnumber index 6b91a92..0c9c7b1 100755 --- a/nextnumber +++ b/nextnumber @@ -8,6 +8,6 @@ fi for i in 1 2 do - last=$(grep -hv "^prop" ./**/*.hs | grep -Ewo "$i[0-9]{3}" | sort -n | tail -n 1) + last=$(grep -hv "^prop" ./**/*.hs | grep -Ewo "${i}[0-9]{3}" | sort -n | tail -n 1) echo "Next ${i}xxx: $((last+1))" done diff --git a/test/check_release b/test/check_release index 5e73d9e..b9cd34b 100755 --- a/test/check_release +++ b/test/check_release @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# shellcheck disable=SC2257 failed=0 fail() { From bd717c9d1be89a3eecd832b73342d2b1afb4dac9 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 1 Apr 2020 22:09:00 -0700 Subject: [PATCH 307/763] Don't warn about [ 0 -ne $FOO ] || [ 0 -ne $BAR ] (fixes #1891) --- src/ShellCheck/Analytics.hs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 8867c5b..a0f5088 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1399,6 +1399,7 @@ prop_checkOrNeq5 = verifyNot checkOrNeq "[[ $a != /home || $a != */public_html/* prop_checkOrNeq6 = verify checkOrNeq "[ $a != a ] || [ $a != b ]" prop_checkOrNeq7 = verify checkOrNeq "[ $a != a ] || [ $a != b ] || true" prop_checkOrNeq8 = verifyNot checkOrNeq "[[ $a != x || $a != x ]]" +prop_checkOrNeq9 = verifyNot checkOrNeq "[ 0 -ne $FOO ] || [ 0 -ne $BAR ]" -- This only catches the most idiomatic cases. Fixme? -- For test-level "or": [ x != y -o x != z ] @@ -1426,9 +1427,17 @@ checkOrNeq _ (T_OrIf id lhs rhs) = sequence_ $ do T_Pipeline _ _ [x] -> getExpr x T_Redirecting _ _ c -> getExpr c T_Condition _ _ c -> getExpr c - TC_Binary _ _ op lhs rhs -> return (lhs, op, rhs) + TC_Binary _ _ op lhs rhs -> orient (lhs, op, rhs) _ -> Nothing + -- Swap items so that the constant side is rhs (or Nothing if both/neither is constant) + orient (lhs, op, rhs) = + case (isConstant lhs, isConstant rhs) of + (True, False) -> return (rhs, op, lhs) + (False, True) -> return (lhs, op, rhs) + _ -> Nothing + + checkOrNeq _ _ = return () From f7547c9a5ad0cec60f7b765881051bf4a56d8a80 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 4 Apr 2020 17:14:02 -0700 Subject: [PATCH 308/763] Stable version v0.7.1 This release is dedicated to the board game Pandemic, for teaching us relevant survival skills like how to stay inside and play board games. --- .multi_arch_docker | 6 ++++++ .travis.yml | 2 +- CHANGELOG.md | 4 +++- ShellCheck.cabal | 2 +- shellcheck.1.md | 4 ++-- 5 files changed, 13 insertions(+), 5 deletions(-) diff --git a/.multi_arch_docker b/.multi_arch_docker index 93c0859..294fed2 100755 --- a/.multi_arch_docker +++ b/.multi_arch_docker @@ -2,6 +2,12 @@ # This script builds and deploys multi-architecture docker images from the # binaries previously built and deployed to GCS by the Travis pipeline. +if [[ "$TRAVIS_SECURE_ENV_VARS" != "true" ]] +then + echo >&2 "Missing TRAVIS_SECURE_ENV_VARS. Skipping Docker builds." + exit 0 +fi + function multi_arch_docker::install_docker_buildx() { # Install up-to-date version of docker, with buildx support. local -r docker_apt_repo='https://download.docker.com/linux/ubuntu' diff --git a/.travis.yml b/.travis.yml index e2dc0bc..0ec9718 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,7 @@ jobs: - stage: Deploy docker image # Deploy only for pushes to master branch, not other branches, not PRs. - if: branch = master AND type = push + if: type = push script: - source ./.multi_arch_docker - set -ex; multi_arch_docker::main; set +x diff --git a/CHANGELOG.md b/CHANGELOG.md index 0dc775a..3e5ca19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## v0.7.1 - soon +## v0.7.1 - 2020-04-04 ### Fixed - `-f diff` no longer claims that it found more issues when it didn't - Known empty variables now correctly trigger SC2086 @@ -7,6 +7,7 @@ called with `builtin` ### Added +- SC1136: Warn about unexpected characters after ]/]] - SC2254: Suggest quoting expansions in case statements - SC2255: Suggest using `$((..))` in `[ 2*3 -eq 6 ]` - SC2256: Warn about translated strings that are known variables @@ -17,6 +18,7 @@ - SC2230: 'command -v' suggestion is now off by default (-i deprecate-which) - SC1081: Keywords are now correctly parsed case sensitively, with a warning + ## v0.7.0 - 2019-07-28 ### Added - Precompiled binaries for macOS and Linux aarch64 diff --git a/ShellCheck.cabal b/ShellCheck.cabal index a848d9c..2254c02 100644 --- a/ShellCheck.cabal +++ b/ShellCheck.cabal @@ -1,5 +1,5 @@ Name: ShellCheck -Version: 0.7.0 +Version: 0.7.1 Synopsis: Shell script analysis tool License: GPL-3 License-file: LICENSE diff --git a/shellcheck.1.md b/shellcheck.1.md index 187d12a..50eaddc 100644 --- a/shellcheck.1.md +++ b/shellcheck.1.md @@ -107,7 +107,7 @@ not warn at all, as `ksh` supports decimals in arithmetic contexts. **-x**,\ **--external-sources** -: Follow 'source' statements even when the file is not specified as input. +: Follow `source` statements even when the file is not specified as input. By default, `shellcheck` will only follow files specified on the command line (plus `/dev/null`). This option allows following any file the script may `source`. @@ -301,7 +301,7 @@ invocation. # RETURN VALUES -ShellCheck uses the follow exit codes: +ShellCheck uses the following exit codes: + 0: All files successfully scanned with no issues. + 1: All files successfully scanned with some issues. From 84d6e53659c44b35a198b3ec80759008ba21c481 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 4 Apr 2020 19:29:28 -0700 Subject: [PATCH 309/763] Update Changelog with new version --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e5ca19..181cc54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +## Git + ## v0.7.1 - 2020-04-04 ### Fixed - `-f diff` no longer claims that it found more issues when it didn't From a30e42ab05cd1c08cc2caddca9d14346c21bbd14 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 4 Apr 2020 19:30:13 -0700 Subject: [PATCH 310/763] Filter GitHub uploads by tag --- .github_deploy | 1 + 1 file changed, 1 insertion(+) diff --git a/.github_deploy b/.github_deploy index 8f648ed..e5d6c3e 100755 --- a/.github_deploy +++ b/.github_deploy @@ -50,6 +50,7 @@ do for file in deploy/* do [[ $file == *.@(xz|gz|zip) ]] || continue + [[ $file == *"$tag"* ]] || continue files+=(-a "$file") done hub release edit "${files[@]}" "$tag" || exit 1 From d2fa88dd91f71f561d151bf46bd3bfb93255be91 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 14:04:23 -0400 Subject: [PATCH 311/763] Simplify nameExpansion --- src/ShellCheck/AnalyzerLib.hs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 453d53d..73449d7 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -840,10 +840,9 @@ getBracedReference s = fromMaybe s $ if c `elem` "*@#?-$!" then return [c] else fail "not special" getSpecial _ = fail "empty" - nameExpansion ('!':rest) = do -- e.g. ${!foo*bar*} - let suffix = dropWhile isVariableChar rest - guard $ suffix /= rest -- e.g. ${!@} - first <- suffix !!! 0 + nameExpansion ('!':next:rest) = do -- e.g. ${!foo*bar*} + guard $ isVariableChar next -- e.g. ${!@} + first <- find (not . isVariableChar) rest guard $ first `elem` "*?" return "" nameExpansion _ = Nothing From 01f442346573361920b5484c0e597716b5e97b06 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 5 Apr 2020 11:38:22 -0700 Subject: [PATCH 312/763] Disable SC2257 about > $((i=42)) for Dash --- src/ShellCheck/Analytics.hs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index a0f5088..f5a8047 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -3596,7 +3596,8 @@ prop_checkModifiedArithmeticInRedirection2 = verify checkModifiedArithmeticInRed prop_checkModifiedArithmeticInRedirection3 = verifyNot checkModifiedArithmeticInRedirection "while true; do true; done > $((i++))" prop_checkModifiedArithmeticInRedirection4 = verify checkModifiedArithmeticInRedirection "cat <<< $((i++))" prop_checkModifiedArithmeticInRedirection5 = verify checkModifiedArithmeticInRedirection "cat << foo\n$((i++))\nfoo\n" -checkModifiedArithmeticInRedirection _ t = +prop_checkModifiedArithmeticInRedirection6 = verifyNot checkModifiedArithmeticInRedirection "#!/bin/dash\nls > $((i=i+1))" +checkModifiedArithmeticInRedirection params t = unless (shellType params == Dash) $ case t of T_Redirecting _ redirs (T_SimpleCommand _ _ (_:_)) -> mapM_ checkRedirs redirs _ -> return () From 2a8170ba05c35b2a31950b2c380e0ad6247121c9 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 15:01:57 -0400 Subject: [PATCH 313/763] Use force instead of reimplementing it --- src/ShellCheck/AnalyzerLib.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 453d53d..086bd7c 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -178,7 +178,7 @@ makeCommentWithFix severity id code str fix = withFix = comment { tcFix = Just fix } - in withFix `deepseq` withFix + in force withFix makeParameters spec = let params = Parameters { From b0dbc79f6973917aa51f1b9629e7a51a03162751 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 15:07:36 -0400 Subject: [PATCH 314/763] Remove unnecessary Maybe from isQuoteFreeElement --- src/ShellCheck/AnalyzerLib.hs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 086bd7c..0a4145a 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -293,15 +293,15 @@ isQuoteFree = isQuoteFreeNode False isQuoteFreeNode strict tree t = - (isQuoteFreeElement t == Just True) || + isQuoteFreeElement t || headOrDefault False (mapMaybe isQuoteFreeContext (drop 1 $ getPath tree t)) where -- Is this node self-quoting in itself? isQuoteFreeElement t = case t of - T_Assignment {} -> return True - T_FdRedirect {} -> return True - _ -> Nothing + T_Assignment {} -> True + T_FdRedirect {} -> True + _ -> False -- Are any subnodes inherently self-quoting? isQuoteFreeContext t = From b3c04ce3d0e0e95a0b3d9be9c58c18426d899d22 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 15:21:44 -0400 Subject: [PATCH 315/763] Implement findFirst in terms of foldr --- src/ShellCheck/AnalyzerLib.hs | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 0a4145a..4348438 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -388,14 +388,13 @@ parents params = getPath (parentMap params) -- Find the first match in a list where the predicate is Just True. -- Stops if it's Just False and ignores Nothing. findFirst :: (a -> Maybe Bool) -> [a] -> Maybe a -findFirst p l = - case l of - [] -> Nothing - (x:xs) -> - case p x of - Just True -> return x - Just False -> Nothing - Nothing -> findFirst p xs +findFirst p = foldr go Nothing + where + go x acc = + case p x of + Just True -> return x + Just False -> Nothing + Nothing -> acc -- Check whether a word is entirely output from a single command tokenIsJustCommandOutput t = case t of From 14ee462ccd572c14ca292b47330e87c9307e0a7c Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 15:27:11 -0400 Subject: [PATCH 316/763] Use execState instead of reimplementing it --- src/ShellCheck/AnalyzerLib.hs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 4348438..8f168dc 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -409,8 +409,7 @@ tokenIsJustCommandOutput t = case t of -- TODO: Replace this with a proper Control Flow Graph getVariableFlow params t = - let (_, stack) = runState (doStackAnalysis startScope endScope t) [] - in reverse stack + reverse $ execState (doStackAnalysis startScope endScope t) [] where startScope t = let scopeType = leadType params t From f55d8c45e567398cfd9effcf08ad6d5706936268 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 15:30:28 -0400 Subject: [PATCH 317/763] Simplify causesSubshell --- src/ShellCheck/AnalyzerLib.hs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 8f168dc..fd0f46d 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -460,11 +460,9 @@ leadType params t = causesSubshell = do (T_Pipeline _ _ list) <- parentPipeline - if length list <= 1 - then return False - else if not $ hasLastpipe params - then return True - else return . not $ (getId . head $ reverse list) == getId t + return $ case list of + _:_:_ -> not (hasLastpipe params) || getId (last list) /= getId t + _ -> False getModifiedVariables t = case t of From f833ee3d5ada7cf4d648fb16ef143e09550b9fac Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 15:35:09 -0400 Subject: [PATCH 318/763] Use a list comprehension instead of a concatMap with extra lists --- src/ShellCheck/AnalyzerLib.hs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index fd0f46d..31ee431 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -467,11 +467,7 @@ leadType params t = getModifiedVariables t = case t of T_SimpleCommand _ vars [] -> - concatMap (\x -> case x of - T_Assignment id _ name _ w -> - [(x, x, name, dataTypeFrom DataString w)] - _ -> [] - ) vars + [(x, x, name, dataTypeFrom DataString w) | x@(T_Assignment id _ name _ w) <- vars] c@T_SimpleCommand {} -> getModifiedVariableCommand c From 67e091674ed35cb20b9053e99e4d38e3d68286e8 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 15:36:10 -0400 Subject: [PATCH 319/763] Remove unnecessary maybeToList The functions we use here are polymorphic enough to work in the [] monad, so there's no point to use them in the Maybe monad and then convert. --- src/ShellCheck/AnalyzerLib.hs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 31ee431..188054c 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -475,7 +475,7 @@ getModifiedVariables t = [(t, v, name, DataString $ SourceFrom [v])] TA_Unary _ "|++" v@(TA_Variable _ name _) -> [(t, v, name, DataString $ SourceFrom [v])] - TA_Assignment _ op (TA_Variable _ name _) rhs -> maybeToList $ do + TA_Assignment _ op (TA_Variable _ name _) rhs -> do guard $ op `elem` ["=", "*=", "/=", "%=", "+=", "-=", "<<=", ">>=", "&=", "^=", "|="] return (t, t, name, DataString $ SourceFrom [rhs]) @@ -487,17 +487,17 @@ getModifiedVariables t = -- Count [[ -v foo ]] as an "assignment". -- This is to prevent [ -v foo ] being unassigned or unused. - TC_Unary id _ "-v" token -> maybeToList $ do + TC_Unary id _ "-v" token -> do str <- fmap (takeWhile (/= '[')) $ -- Quoted index flip getLiteralStringExt token $ \x -> case x of T_Glob _ s -> return s -- Unquoted index - _ -> Nothing + _ -> [] guard . not . null $ str return (t, token, str, DataString SourceChecked) - T_DollarBraced _ _ l -> maybeToList $ do + T_DollarBraced _ _ l -> do let string = bracedString t let modifier = getBracedModifier string guard $ any (`isPrefixOf` modifier) ["=", ":="] @@ -747,9 +747,9 @@ getReferencedVariables parents t = literalizer t = case t of T_Glob _ s -> return s -- Also when parsed as globs - _ -> Nothing + _ -> [] - getIfReference context token = maybeToList $ do + getIfReference context token = do str@(h:_) <- getLiteralStringExt literalizer token when (isDigit h) $ fail "is a number" return (context, token, getBracedReference str) From f109f9ab9270dd78bd8f8dc2158c9ce4904f599e Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 15:38:16 -0400 Subject: [PATCH 320/763] Remove unnecessary as-patterns --- src/ShellCheck/AnalyzerLib.hs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 188054c..543f6e6 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -468,8 +468,8 @@ getModifiedVariables t = case t of T_SimpleCommand _ vars [] -> [(x, x, name, dataTypeFrom DataString w) | x@(T_Assignment id _ name _ w) <- vars] - c@T_SimpleCommand {} -> - getModifiedVariableCommand c + T_SimpleCommand {} -> + getModifiedVariableCommand t TA_Unary _ "++|" v@(TA_Variable _ name _) -> [(t, v, name, DataString $ SourceFrom [v])] @@ -503,10 +503,10 @@ getModifiedVariables t = guard $ any (`isPrefixOf` modifier) ["=", ":="] return (t, t, getBracedReference string, DataString $ SourceFrom [l]) - t@(T_FdRedirect _ ('{':var) op) -> -- {foo}>&2 modifies foo + T_FdRedirect _ ('{':var) op -> -- {foo}>&2 modifies foo [(t, t, takeWhile (/= '}') var, DataString SourceInteger) | not $ isClosingFileOp op] - t@(T_CoProc _ name _) -> + T_CoProc _ name _ -> [(t, t, fromMaybe "COPROC" name, DataArray SourceInteger)] --Points to 'for' rather than variable @@ -730,7 +730,7 @@ getReferencedVariables parents t = (t, t, "output") ] - t@(T_FdRedirect _ ('{':var) op) -> -- {foo}>&- references and closes foo + T_FdRedirect _ ('{':var) op -> -- {foo}>&- references and closes foo [(t, t, takeWhile (/= '}') var) | isClosingFileOp op] x -> getReferencedVariableCommand x where From e4eb2d157f8211e14b66d0718fa4d90e3fefbd8a Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 16:03:49 -0400 Subject: [PATCH 321/763] Remove an unnecessary operator section --- src/ShellCheck/AnalyzerLib.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 543f6e6..4cd4b5e 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -564,7 +564,7 @@ getModifiedVariableCommand base@(T_SimpleCommand id cmdPrefix (T_NormalWord _ (T let params = map getLiteral rest readArrayVars = getReadArrayVariables rest in - catMaybes . (++ readArrayVars) . takeWhile isJust . reverse $ params + catMaybes $ takeWhile isJust (reverse params) ++ readArrayVars "getopts" -> case rest of opts:var:_ -> maybeToList $ getLiteral var From 2ebf522a52efe4ef092009f0db6da4cdd25edf5b Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 16:07:19 -0400 Subject: [PATCH 322/763] Simplify isArrayFlag --- src/ShellCheck/AnalyzerLib.hs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 4cd4b5e..8208a5b 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -669,12 +669,10 @@ getModifiedVariableCommand base@(T_SimpleCommand id cmdPrefix (T_NormalWord _ (T map (getLiteralArray . snd) (filter (isArrayFlag . fst) (zip args (tail args))) - isArrayFlag x = fromMaybe False $ do - str <- getLiteralString x - return $ case str of - '-':'-':_ -> False - '-':str -> 'a' `elem` str - _ -> False + isArrayFlag x = case getLiteralString x of + Just ('-':'-':_) -> False + Just ('-':str) -> 'a' `elem` str + _ -> False -- get the FLAGS_ variable created by a shflags DEFINE_ call getFlagVariable (n:v:_) = do From 4604066c3729766b49da0251786286b6c1ae8caa Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 16:16:12 -0400 Subject: [PATCH 323/763] Use head instead of (!! 0) --- src/ShellCheck/AnalyzerLib.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 8208a5b..c5eaea6 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -798,7 +798,7 @@ getVariablesFromLiteralToken token = prop_getVariablesFromLiteral1 = getVariablesFromLiteral "$foo${bar//a/b}$BAZ" == ["foo", "bar", "BAZ"] getVariablesFromLiteral string = - map (!! 0) $ matchAllSubgroups variableRegex string + map head $ matchAllSubgroups variableRegex string where variableRegex = mkRegex "\\$\\{?([A-Za-z0-9_]+)" From 1cf0aa25e9a76585f34798628cf7cc7d35642970 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 16:19:18 -0400 Subject: [PATCH 324/763] Simplify dropPrefix --- src/ShellCheck/AnalyzerLib.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index c5eaea6..26e65e6 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -820,8 +820,8 @@ getBracedReference s = fromMaybe s $ nameExpansion s `mplus` takeName noPrefix `mplus` getSpecial noPrefix `mplus` getSpecial s where noPrefix = dropPrefix s - dropPrefix (c:rest) = if c `elem` "!#" then rest else c:rest - dropPrefix "" = "" + dropPrefix (c:rest) | c `elem` "!#" = rest + dropPrefix cs = cs takeName s = do let name = takeWhile isVariableChar s guard . not $ null name From ca41440a678879d586320250a4b91913d1256a6e Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 16:21:07 -0400 Subject: [PATCH 325/763] Simplify getSpecial --- src/ShellCheck/AnalyzerLib.hs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 26e65e6..95b236a 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -826,9 +826,8 @@ getBracedReference s = fromMaybe s $ let name = takeWhile isVariableChar s guard . not $ null name return name - getSpecial (c:_) = - if c `elem` "*@#?-$!" then return [c] else fail "not special" - getSpecial _ = fail "empty" + getSpecial (c:_) | c `elem` "*@#?-$!" = return [c] + getSpecial _ = fail "empty or not special" nameExpansion ('!':rest) = do -- e.g. ${!foo*bar*} let suffix = dropWhile isVariableChar rest From 0cc5ed4563ccd87b200dd41c2801f8d4d79b8f15 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 16:25:43 -0400 Subject: [PATCH 326/763] Don't bother with asks if you're just immediately binding the result anyway --- src/ShellCheck/AnalyzerLib.hs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 95b236a..32de46a 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -354,8 +354,8 @@ getClosestCommand tree t = -- Like above, if koala_man knew Haskell when starting this project. getClosestCommandM t = do - tree <- asks parentMap - return $ getClosestCommand tree t + params <- ask + return $ getClosestCommand (parentMap params) t -- Is the token used as a command name (the first word in a T_SimpleCommand)? usedAsCommandName tree token = go (getId token) (tail $ getPath tree token) @@ -377,8 +377,8 @@ getPath tree t = t : -- Version of the above taking the map from the current context -- Todo: give this the name "getPath" getPathM t = do - map <- asks parentMap - return $ getPath map t + params <- ask + return $ getPath (parentMap params) t isParentOf tree parent child = elem (getId parent) . map getId $ getPath tree child @@ -866,8 +866,8 @@ headOrDefault def _ = def -- Run a command if the shell is in the given list whenShell l c = do - shell <- asks shellType - when (shell `elem` l ) c + params <- ask + when (shellType params `elem` l ) c filterByAnnotation asSpec params = From fb55072302c14d4785875357d4400de736f4eeb9 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 16:30:59 -0400 Subject: [PATCH 327/763] Implement supportsArrays with pattern-matching --- src/ShellCheck/AnalyzerLib.hs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 32de46a..a3fa29d 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -934,7 +934,9 @@ getOpts string flags = process flags more <- process rest2 return $ (flag1, token1) : more -supportsArrays shell = shell == Bash || shell == Ksh +supportsArrays Bash = True +supportsArrays Ksh = True +supportsArrays _ = False -- Returns true if the shell is Bash or Ksh (sorry for the name, Ksh) isBashLike :: Parameters -> Bool From d22e0aa4a79ddd8182c81898b790c9b9bb850937 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 16:38:52 -0400 Subject: [PATCH 328/763] Simplify process Note to self: This is a lot like foldr or traverse, and would be trivial to implement as such if it didn't need to peek ahead when takesArg is true. I wonder if there's a clean way to implement it in terms of one of them anyway. --- src/ShellCheck/AnalyzerLib.hs | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index a3fa29d..7c2440b 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -919,20 +919,15 @@ getOpts string flags = process flags flagMap = Map.fromList $ ("", False) : flagList string process [] = return [] - process [(token, flag)] = do + process ((token1, flag):rest1) = do takesArg <- Map.lookup flag flagMap - guard $ not takesArg - return [(flag, token)] - process ((token1, flag1):rest2@((token2, flag2):rest)) = do - takesArg <- Map.lookup flag1 flagMap - if takesArg - then do - guard $ null flag2 - more <- process rest - return $ (flag1, token2) : more - else do - more <- process rest2 - return $ (flag1, token1) : more + (token, rest) <- if takesArg + then case rest1 of + (token2, ""):rest2 -> return (token2, rest2) + _ -> fail "takesArg without valid arg" + else return (token1, rest1) + more <- process rest + return $ (flag, token) : more supportsArrays Bash = True supportsArrays Ksh = True From 8f105074feebda422e4242d04bb4afdf1e0431cf Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 19:01:56 -0400 Subject: [PATCH 329/763] Simplify getCommandNameAndToken --- src/ShellCheck/ASTLib.hs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 2b40705..05c3838 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -317,13 +317,11 @@ getCommandNameAndToken :: Token -> (Maybe String, Token) getCommandNameAndToken t = fromMaybe (Nothing, t) $ do (T_SimpleCommand _ _ (w:rest)) <- getCommand t s <- getLiteralString w - if "busybox" `isSuffixOf` s || "builtin" == s - then - case rest of - (applet:_) -> return (getLiteralString applet, applet) - _ -> return (Just s, w) - else - return (Just s, w) + return $ case rest of + (applet:_) | "busybox" `isSuffixOf` s || "builtin" == s -> + (getLiteralString applet, applet) + _ -> + (Just s, w) -- If a command substitution is a single command, get its name. From b6cff5ea0e1626ad3251ffa018b5990d3b7c23c5 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 19:06:16 -0400 Subject: [PATCH 330/763] Simplify getAssociativeArrays --- src/ShellCheck/ASTLib.hs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 05c3838..9494150 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -398,10 +398,10 @@ getAssociativeArrays t = f t@T_SimpleCommand {} = sequence_ $ do name <- getCommandName t let assocNames = ["declare","local","typeset"] - guard $ elem name assocNames + guard $ name `elem` assocNames let flags = getAllFlags t - guard $ elem "A" $ map snd flags - let args = map fst . filter ((==) "" . snd) $ flags + guard $ "A" `elem` map snd flags + let args = [arg | (arg, "") <- flags] let names = mapMaybe (getLiteralStringExt nameAssignments) args return $ tell names f _ = return () From 322842b57e370d484355172497c69bd163184ef4 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 19:29:40 -0400 Subject: [PATCH 331/763] Remove unnecessary monadicity from wordToPseudoGlob --- src/ShellCheck/ASTLib.hs | 27 +++++++++++++-------------- src/ShellCheck/Analytics.hs | 16 +++++----------- 2 files changed, 18 insertions(+), 25 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 9494150..758d79c 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -419,25 +419,25 @@ data PseudoGlob = PGAny | PGMany | PGChar Char -- Turn a word into a PG pattern, replacing all unknown/runtime values with -- PGMany. -wordToPseudoGlob :: Token -> Maybe [PseudoGlob] +wordToPseudoGlob :: Token -> [PseudoGlob] wordToPseudoGlob word = - simplifyPseudoGlob . concat <$> mapM f (getWordParts word) + simplifyPseudoGlob . concatMap f $ getWordParts word where f x = case x of - T_Literal _ s -> return $ map PGChar s - T_SingleQuoted _ s -> return $ map PGChar s + T_Literal _ s -> map PGChar s + T_SingleQuoted _ s -> map PGChar s - T_DollarBraced {} -> return [PGMany] - T_DollarExpansion {} -> return [PGMany] - T_Backticked {} -> return [PGMany] + T_DollarBraced {} -> [PGMany] + T_DollarExpansion {} -> [PGMany] + T_Backticked {} -> [PGMany] - T_Glob _ "?" -> return [PGAny] - T_Glob _ ('[':_) -> return [PGAny] - T_Glob {} -> return [PGMany] + T_Glob _ "?" -> [PGAny] + T_Glob _ ('[':_) -> [PGAny] + T_Glob {} -> [PGMany] - T_Extglob {} -> return [PGMany] + T_Extglob {} -> [PGMany] - _ -> return [PGMany] + _ -> [PGMany] -- Turn a word into a PG pattern, but only if we can preserve -- exact semantics. @@ -500,8 +500,7 @@ pseudoGlobIsSuperSetof = matchable matchable (PGMany : rest) [] = matchable rest [] matchable _ _ = False -wordsCanBeEqual x y = fromMaybe True $ - liftM2 pseudoGlobsCanOverlap (wordToPseudoGlob x) (wordToPseudoGlob y) +wordsCanBeEqual x y = pseudoGlobsCanOverlap (wordToPseudoGlob x) (wordToPseudoGlob y) -- Is this an expansion that can be quoted, -- e.g. $(foo) `foo` $foo (but not {foo,})? diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index f5a8047..599e257 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -3137,9 +3137,7 @@ checkUnmatchableCases params t = if isConstant word then warn (getId word) 2194 "This word is constant. Did you forget the $ on a variable?" - else sequence_ $ do - pg <- wordToPseudoGlob word - return $ mapM_ (check pg) allpatterns + else mapM_ (check $ wordToPseudoGlob word) allpatterns let exactGlobs = tupMap wordToExactPseudoGlob breakpatterns let fuzzyGlobs = tupMap wordToPseudoGlob breakpatterns @@ -3152,15 +3150,13 @@ checkUnmatchableCases params t = fst3 (x,_,_) = x snd3 (_,x,_) = x tp = tokenPositions params - check target candidate = sequence_ $ do - candidateGlob <- wordToPseudoGlob candidate - guard . not $ pseudoGlobsCanOverlap target candidateGlob - return $ warn (getId candidate) 2195 - "This pattern will never match the case statement's word. Double check them." + check target candidate = unless (pseudoGlobsCanOverlap target $ wordToPseudoGlob candidate) $ + warn (getId candidate) 2195 + "This pattern will never match the case statement's word. Double check them." tupMap f l = map (\x -> (x, f x)) l checkDoms ((glob, Just x), rest) = - forM_ (find (\(_, p) -> x `pseudoGlobIsSuperSetof` p) valids) $ + forM_ (find (\(_, p) -> x `pseudoGlobIsSuperSetof` p) rest) $ \(first,_) -> do warn (getId glob) 2221 $ "This pattern always overrides a later one" <> patternContext (getId first) warn (getId first) 2222 $ "This pattern never matches because of a previous pattern" <> patternContext (getId glob) @@ -3170,8 +3166,6 @@ checkUnmatchableCases params t = case posLine . fst <$> Map.lookup id tp of Just l -> " on line " <> show l <> "." _ -> "." - - valids = [(x,y) | (x, Just y) <- rest] checkDoms _ = return () From 0f9b0f18a435cafc7cb30451c85ee6efbef029ec Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 19:30:21 -0400 Subject: [PATCH 332/763] Remove unnecessary cases from wordToPseudoGlob --- src/ShellCheck/ASTLib.hs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 758d79c..da91d09 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -427,15 +427,8 @@ wordToPseudoGlob word = T_Literal _ s -> map PGChar s T_SingleQuoted _ s -> map PGChar s - T_DollarBraced {} -> [PGMany] - T_DollarExpansion {} -> [PGMany] - T_Backticked {} -> [PGMany] - T_Glob _ "?" -> [PGAny] T_Glob _ ('[':_) -> [PGAny] - T_Glob {} -> [PGMany] - - T_Extglob {} -> [PGMany] _ -> [PGMany] From d45ab327b0ca4fe949c13b95fc7eebb8c8d74a5d Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 19:45:28 -0400 Subject: [PATCH 333/763] Only perform the comparisons once --- src/ShellCheck/Fixer.hs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/ShellCheck/Fixer.hs b/src/ShellCheck/Fixer.hs index 16bdd98..1409b24 100644 --- a/src/ShellCheck/Fixer.hs +++ b/src/ShellCheck/Fixer.hs @@ -273,10 +273,10 @@ getPrefixSum = f 0 where f sum _ PSLeaf = sum f sum target (PSBranch pivot left right cumulative) = - case () of - _ | target < pivot -> f sum target left - _ | target > pivot -> f (sum+cumulative) target right - _ -> sum+cumulative + case target `compare` pivot of + LT -> f sum target left + GT -> f (sum+cumulative) target right + EQ -> sum+cumulative -- Add a value to the Prefix Sum tree at the given index. -- Values accumulate: addPSValue 42 2 . addPSValue 42 3 == addPSValue 42 5 @@ -285,10 +285,10 @@ addPSValue key value tree = if value == 0 then tree else f tree where f PSLeaf = PSBranch key PSLeaf PSLeaf value f (PSBranch pivot left right sum) = - case () of - _ | key < pivot -> PSBranch pivot (f left) right (sum + value) - _ | key > pivot -> PSBranch pivot left (f right) sum - _ -> PSBranch pivot left right (sum + value) + case key `compare` pivot of + LT -> PSBranch pivot (f left) right (sum + value) + GT -> PSBranch pivot left (f right) sum + EQ -> PSBranch pivot left right (sum + value) prop_pstreeSumsCorrectly kvs targets = let From 773e98868daf2138b249d805e2297feeda32827a Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 19:53:40 -0400 Subject: [PATCH 334/763] Use foldr in checkFindNameGlob --- src/ShellCheck/Checks/Commands.hs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index e321102..e0a2749 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -199,12 +199,11 @@ prop_checkFindNameGlob3 = verifyNot checkFindNameGlob "find * -name '*.php'" checkFindNameGlob = CommandCheck (Basename "find") (f . arguments) where acceptsGlob s = s `elem` [ "-ilname", "-iname", "-ipath", "-iregex", "-iwholename", "-lname", "-name", "-path", "-regex", "-wholename" ] f [] = return () - f (x:xs) = g x xs - g _ [] = return () - g a (b:r) = do + f (x:xs) = foldr g (const $ return ()) xs x + g b acc a = do forM_ (getLiteralString a) $ \s -> when (acceptsGlob s && isGlob b) $ warn (getId b) 2061 $ "Quote the parameter to " ++ s ++ " so the shell won't interpret it." - g b r + acc b prop_checkNeedlessExpr = verify checkNeedlessExpr "foo=$(expr 3 + 2)" From 9027a9239fafe40463dba495bd25fdd739a4dd52 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 20:03:17 -0400 Subject: [PATCH 335/763] Use pattern matching instead of snd --- src/ShellCheck/Checks/Commands.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index e0a2749..0d887dd 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -461,8 +461,8 @@ checkMkdirDashPM = CommandCheck (Basename "mkdir") check where check t = sequence_ $ do let flags = getAllFlags t - dashP <- find ((\f -> f == "p" || f == "parents") . snd) flags - dashM <- find ((\f -> f == "m" || f == "mode") . snd) flags + dashP <- find (\(_,f) -> f == "p" || f == "parents") flags + dashM <- find (\(_,f) -> f == "m" || f == "mode") flags -- mkdir -pm 0700 dir is fine, so is ../dir, but dir/subdir is not. guard $ any couldHaveSubdirs (drop 1 $ arguments t) return $ warn (getId $ fst dashM) 2174 "When used with -p, -m only applies to the deepest directory." From e8501151dd5d4ad90687deefa434f424701b6c7c Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 20:04:54 -0400 Subject: [PATCH 336/763] Use a guard instead of unless --- src/ShellCheck/Checks/Commands.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 0d887dd..9e19ab5 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -482,7 +482,7 @@ prop_checkNonportableSignals7 = verifyNot checkNonportableSignals "trap 'stop' i checkNonportableSignals = CommandCheck (Exactly "trap") (f . arguments) where f args = case args of - first:rest -> unless (isFlag first) $ mapM_ check rest + first:rest | not $ isFlag first -> mapM_ check rest _ -> return () check param = sequence_ $ do From fa841cb2701863b84ace7d652ff977db315e599c Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 20:08:02 -0400 Subject: [PATCH 337/763] Prefer pattern matching in undirected --- src/ShellCheck/Checks/Commands.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 9e19ab5..19884ee 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -519,9 +519,9 @@ checkInteractiveSu = CommandCheck (Basename "su") f info (getId cmd) 2117 "To run commands as another user, use su -c or sudo." - undirected (T_Pipeline _ _ l) = length l <= 1 + undirected (T_Pipeline _ _ (_:_:_)) = False -- This should really just be modifications to stdin, but meh - undirected (T_Redirecting _ list _) = null list + undirected (T_Redirecting _ (_:_) _) = False undirected _ = True From 9747b1d5c3961fbcadf657dffe324b66dc5856a3 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 20:10:56 -0400 Subject: [PATCH 338/763] Simplify checkArg --- src/ShellCheck/Checks/Commands.hs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 19884ee..1633dc6 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -538,9 +538,8 @@ checkSshCommandString = CommandCheck (Basename "ssh") (f . arguments) ([], hostport:r@(_:_)) -> checkArg $ last r _ -> return () checkArg (T_NormalWord _ [T_DoubleQuoted id parts]) = - case filter (not . isConstant) parts of - [] -> return () - (x:_) -> info (getId x) 2029 + forM_ (find (not . isConstant) parts) $ + \x -> info (getId x) 2029 "Note that, unescaped, this expands on the client side." checkArg _ = return () From df4928f4e3191e18abbaa5085c13c790eeb694fe Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 20:14:03 -0400 Subject: [PATCH 339/763] Use MultiWayIf instead of case-matching on () --- src/ShellCheck/Checks/Commands.hs | 32 +++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 1633dc6..440b74a 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -19,6 +19,7 @@ -} {-# LANGUAGE TemplateHaskell #-} {-# LANGUAGE FlexibleContexts #-} +{-# LANGUAGE MultiWayIf #-} -- This module contains checks that examine specific commands by name. module ShellCheck.Checks.Commands (checker, optionalChecks, ShellCheck.Checks.Commands.runTests) where @@ -578,22 +579,21 @@ checkPrintfVar = CommandCheck (Exactly "printf") (f . arguments) where let formatCount = length formats let argCount = length more - return $ - case () of - () | argCount == 0 && formatCount == 0 -> - return () -- This is fine - () | formatCount == 0 && argCount > 0 -> - err (getId format) 2182 - "This printf format string has no variables. Other arguments are ignored." - () | any mayBecomeMultipleArgs more -> - return () -- We don't know so trust the user - () | argCount < formatCount && onlyTrailingTs formats argCount -> - return () -- Allow trailing %()Ts since they use the current time - () | argCount > 0 && argCount `mod` formatCount == 0 -> - return () -- Great: a suitable number of arguments - () -> - warn (getId format) 2183 $ - "This format string has " ++ show formatCount ++ " variables, but is passed " ++ show argCount ++ " arguments." + return $ if + | argCount == 0 && formatCount == 0 -> + return () -- This is fine + | formatCount == 0 && argCount > 0 -> + err (getId format) 2182 + "This printf format string has no variables. Other arguments are ignored." + | any mayBecomeMultipleArgs more -> + return () -- We don't know so trust the user + | argCount < formatCount && onlyTrailingTs formats argCount -> + return () -- Allow trailing %()Ts since they use the current time + | argCount > 0 && argCount `mod` formatCount == 0 -> + return () -- Great: a suitable number of arguments + | otherwise -> + warn (getId format) 2183 $ + "This format string has " ++ show formatCount ++ " variables, but is passed " ++ show argCount ++ " arguments." unless ('%' `elem` concat (oversimplify format) || isLiteral format) $ info (getId format) 2059 From cfa2a663afd83794edacd81ffc0ef9f972e5be26 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 20:21:14 -0400 Subject: [PATCH 340/763] Simplify checkSetAssignment --- src/ShellCheck/Checks/Commands.hs | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 440b74a..4f92fa9 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -661,17 +661,12 @@ prop_checkSetAssignment5 = verifyNot checkSetAssignment "set 'a=5'" prop_checkSetAssignment6 = verifyNot checkSetAssignment "set" checkSetAssignment = CommandCheck (Exactly "set") (f . arguments) where - f (var:value:rest) = - let str = literal var in - when (isVariableName str || isAssignment str) $ - msg (getId var) - f (var:_) = - when (isAssignment $ literal var) $ - msg (getId var) + f (var:rest) + | (not (null rest) && isVariableName str) || isAssignment str = + warn (getId var) 2121 "To assign a variable, use just 'var=value', no 'set ..'." + where str = literal var f _ = return () - msg id = warn id 2121 "To assign a variable, use just 'var=value', no 'set ..'." - isAssignment str = '=' `elem` str literal (T_NormalWord _ l) = concatMap literal l literal (T_Literal _ str) = str From ed331b816b083d8598de3e4ea77a27f2f2daa83a Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 20:32:39 -0400 Subject: [PATCH 341/763] Simplify warnRedundant --- src/ShellCheck/Checks/Commands.hs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 4f92fa9..c68bdd1 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -878,10 +878,10 @@ checkWhileGetoptsCase = CommandCheck (Exactly "getopts") f warnUnhandled optId caseId str = warn caseId 2213 $ "getopts specified -" ++ str ++ ", but it's not handled by this 'case'." - warnRedundant (key, expr) = sequence_ $ do - str <- key - guard $ str `notElem` ["*", ":", "?"] - return $ warn (getId expr) 2214 "This case is not specified by getopts." + warnRedundant (Just str, expr) + | str `notElem` ["*", ":", "?"] = + warn (getId expr) 2214 "This case is not specified by getopts." + warnRedundant _ = return () getHandledStrings (_, globs, _) = map (\x -> (literal x, x)) globs From 5084ba8d7ed017551de498365861f8a585eca360 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 21:23:34 -0400 Subject: [PATCH 342/763] Make skipRepeating lazier and faster --- src/ShellCheck/Checks/Commands.hs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index e321102..14a8c09 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -966,9 +966,9 @@ checkCatastrophicRm = CommandCheck (Basename "rm") $ \t -> f _ = return "" stripTrailing c = reverse . dropWhile (== c) . reverse - skipRepeating c (a:b:rest) | a == b && b == c = skipRepeating c (b:rest) - skipRepeating c (a:r) = a:skipRepeating c r - skipRepeating _ [] = [] + skipRepeating c = foldr go [] + where + go a r = a : if a == c then case r of b:rest | b == c -> rest; _ -> r else r paths = [ "", "/bin", "/etc", "/home", "/mnt", "/usr", "/usr/share", "/usr/local", From cd38afce26d9914085ea632d3eebded5fd667778 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 21:45:51 -0400 Subject: [PATCH 343/763] Make it slightly lazier still (and more clear) --- src/ShellCheck/Checks/Commands.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 14a8c09..ae02cd8 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -968,7 +968,7 @@ checkCatastrophicRm = CommandCheck (Basename "rm") $ \t -> stripTrailing c = reverse . dropWhile (== c) . reverse skipRepeating c = foldr go [] where - go a r = a : if a == c then case r of b:rest | b == c -> rest; _ -> r else r + go a r = a : case r of b:rest | b == c && a == b -> rest; _ -> r paths = [ "", "/bin", "/etc", "/home", "/mnt", "/usr", "/usr/share", "/usr/local", From facf0d1e27185861f7ede9ac1bd9803b440d6c77 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 21:59:27 -0400 Subject: [PATCH 344/763] Write getLiteralArgs with foldr and without fromMaybe or monads --- src/ShellCheck/Checks/ShellSupport.hs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 49d3212..de5f78c 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -289,10 +289,11 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do -- Get the literal options from a list of arguments, -- up until the first non-literal one getLiteralArgs :: [Token] -> [(Id, String)] - getLiteralArgs (first:rest) = fromMaybe [] $ do - str <- getLiteralString first - return $ (getId first, str) : getLiteralArgs rest - getLiteralArgs [] = [] + getLiteralArgs = foldr go [] + where + go first rest = case getLiteralString first of + Just str -> (getId first, str) : rest + Nothing -> [] -- Check a flag-option pair (such as -o errexit) checkOptions (flag@(fid,flag') : opt@(oid,opt') : rest) From 8a6679fd8a12e365411d97918206f8dc1db5c1ec Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 22:03:50 -0400 Subject: [PATCH 345/763] Remove unnecessary fromMaybe and when from bashism --- src/ShellCheck/Checks/ShellSupport.hs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index de5f78c..054661a 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -391,11 +391,10 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do ("unset", Just ["f", "v"]), ("wait", Just []) ] - bashism t@(T_SourceCommand id src _) = - let name = fromMaybe "" $ getCommandName src - in when (name == "source") $ warnMsg id "'source' in place of '.' is" - bashism (TA_Expansion _ (T_Literal id str : _)) | str `matches` radix = - when (str `matches` radix) $ warnMsg id "arithmetic base conversion is" + bashism t@(T_SourceCommand id src _) + | getCommandName src == Just "source" = warnMsg id "'source' in place of '.' is" + bashism (TA_Expansion _ (T_Literal id str : _)) + | str `matches` radix = warnMsg id "arithmetic base conversion is" where radix = mkRegex "^[0-9]+#" bashism _ = return () From 64c31d91422e3c48cc7ed0176b069fa11456cee8 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 22:13:39 -0400 Subject: [PATCH 346/763] Use fromRight instead of reimplementing it --- src/ShellCheck/Formatter/GCC.hs | 3 ++- src/ShellCheck/Formatter/JSON1.hs | 3 ++- src/ShellCheck/Formatter/TTY.hs | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/ShellCheck/Formatter/GCC.hs b/src/ShellCheck/Formatter/GCC.hs index 9c5fa5f..a7307a9 100644 --- a/src/ShellCheck/Formatter/GCC.hs +++ b/src/ShellCheck/Formatter/GCC.hs @@ -22,6 +22,7 @@ module ShellCheck.Formatter.GCC (format) where import ShellCheck.Interface import ShellCheck.Formatter.Format +import Data.Either import Data.List import GHC.Exts import System.IO @@ -44,7 +45,7 @@ outputAll cr sys = mapM_ f groups f group = do let filename = sourceFile (head group) result <- (siReadFile sys) filename - let contents = either (const "") id result + let contents = fromRight "" result outputResult filename contents group outputResult filename contents warnings = do diff --git a/src/ShellCheck/Formatter/JSON1.hs b/src/ShellCheck/Formatter/JSON1.hs index 7335d8c..edc1974 100644 --- a/src/ShellCheck/Formatter/JSON1.hs +++ b/src/ShellCheck/Formatter/JSON1.hs @@ -24,6 +24,7 @@ import ShellCheck.Interface import ShellCheck.Formatter.Format import Data.Aeson +import Data.Either import Data.IORef import Data.Monoid import GHC.Exts @@ -118,7 +119,7 @@ collectResult ref cr sys = mapM_ f groups f group = do let filename = sourceFile (head group) result <- siReadFile sys filename - let contents = either (const "") id result + let contents = fromRight "" result let comments' = makeNonVirtual comments contents modifyIORef ref (\x -> comments' ++ x) diff --git a/src/ShellCheck/Formatter/TTY.hs b/src/ShellCheck/Formatter/TTY.hs index 4dabf45..ddf36aa 100644 --- a/src/ShellCheck/Formatter/TTY.hs +++ b/src/ShellCheck/Formatter/TTY.hs @@ -25,6 +25,7 @@ import ShellCheck.Formatter.Format import Control.Monad import Data.Array +import Data.Either import Data.Foldable import Data.Ord import Data.IORef @@ -122,7 +123,7 @@ outputResult options ref result sys = do outputForFile color sys comments = do let fileName = sourceFile (head comments) result <- (siReadFile sys) fileName - let contents = either (const "") id result + let contents = fromRight "" result let fileLinesList = lines contents let lineCount = length fileLinesList let fileLines = listArray (1, lineCount) fileLinesList From 1c6202dba4d31feb1a4e483df849a74f39e790e3 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 22:25:19 -0400 Subject: [PATCH 347/763] Avoid some awkward parentheses with forM_ --- src/ShellCheck/Formatter/TTY.hs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/ShellCheck/Formatter/TTY.hs b/src/ShellCheck/Formatter/TTY.hs index ddf36aa..44bda1e 100644 --- a/src/ShellCheck/Formatter/TTY.hs +++ b/src/ShellCheck/Formatter/TTY.hs @@ -128,7 +128,7 @@ outputForFile color sys comments = do let lineCount = length fileLinesList let fileLines = listArray (1, lineCount) fileLinesList let groups = groupWith lineNo comments - mapM_ (\commentsForLine -> do + forM_ groups $ \commentsForLine -> do let lineNum = fromIntegral $ lineNo (head commentsForLine) let line = if lineNum < 1 || lineNum > lineCount then "" @@ -137,10 +137,9 @@ outputForFile color sys comments = do putStrLn $ color "message" $ "In " ++ fileName ++" line " ++ show lineNum ++ ":" putStrLn (color "source" line) - mapM_ (\c -> putStrLn (color (severityText c) $ cuteIndent c)) commentsForLine + forM_ commentsForLine $ \c -> putStrLn $ color (severityText c) $ cuteIndent c putStrLn "" showFixedString color commentsForLine (fromIntegral lineNum) fileLines - ) groups -- Pick out only the lines necessary to show a fix in action sliceFile :: Fix -> Array Int String -> (Fix, Array Int String) From 3e17a2096595f004534ae29920be60f239e31fab Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sat, 11 Apr 2020 17:29:28 -0400 Subject: [PATCH 348/763] Simplify thenSkip, and use in another location --- src/ShellCheck/Parser.hs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 64d75c6..b96aee7 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -373,10 +373,7 @@ parseNoteAtId id c l a = do parseNoteAtWithEnd start end c l a = addParseNote $ ParseNote start end c l a --------- Convenient combinators -thenSkip main follow = do - r <- main - optional follow - return r +thenSkip main follow = main <* optional follow unexpecting s p = try $ (try p >> fail ("Unexpected " ++ s)) <|> return () @@ -420,7 +417,7 @@ acceptButWarn parser level code note = parsecBracket before after op = do val <- before - (op val <* optional (after val)) <|> (after val *> fail "") + op val `thenSkip` after val <|> (after val *> fail "") swapContext contexts p = parsecBracket (getCurrentContexts <* setCurrentContexts contexts) From 8a7497c4f02304811b8ce5ec15d3263fc78ba83e Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 17:07:05 -0400 Subject: [PATCH 349/763] Simplify checkVariableBraces --- src/ShellCheck/Analytics.hs | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index f5a8047..dd4e4c7 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1955,19 +1955,16 @@ prop_CheckVariableBraces1 = verify checkVariableBraces "a='123'; echo $a" prop_CheckVariableBraces2 = verifyNot checkVariableBraces "a='123'; echo ${a}" prop_CheckVariableBraces3 = verifyNot checkVariableBraces "#shellcheck disable=SC2016\necho '$a'" prop_CheckVariableBraces4 = verifyNot checkVariableBraces "echo $* $1" -checkVariableBraces params t = - case t of - T_DollarBraced id False _ - | name `notElem` unbracedVariables -> - styleWithFix id 2250 - "Prefer putting braces around variable references even when not strictly required." - (fixFor t) - - _ -> return () +checkVariableBraces params t@(T_DollarBraced id False _) + | name `notElem` unbracedVariables = + styleWithFix id 2250 + "Prefer putting braces around variable references even when not strictly required." + (fixFor t) where name = getBracedReference $ bracedString t fixFor token = fixWith [replaceStart (getId token) params 1 "${" ,replaceEnd (getId token) params 0 "}"] +checkVariableBraces _ _ = return () prop_checkQuotesInLiterals1 = verifyTree checkQuotesInLiterals "param='--foo=\"bar\"'; app $param" prop_checkQuotesInLiterals1a= verifyTree checkQuotesInLiterals "param=\"--foo='lolbar'\"; app $param" From a9d564a8bc8bbdd624543b2e06082e9bdc838a1b Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 17:19:14 -0400 Subject: [PATCH 350/763] Combine bracedString into getSingleUnmodifiedVariable Everywhere we used getSingleUnmodifiedVariable, we just called bracedString on the result. Move this into that function instead, and rename it accordingly. --- src/ShellCheck/Checks/Commands.hs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index e321102..4e9c045 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -687,8 +687,7 @@ prop_checkExportedExpansions4 = verifyNot checkExportedExpansions "export ${foo? checkExportedExpansions = CommandCheck (Exactly "export") (mapM_ check . arguments) where check t = sequence_ $ do - var <- getSingleUnmodifiedVariable t - let name = bracedString var + name <- getSingleUnmodifiedBracedString t return . warn (getId t) 2163 $ "This does not export '" ++ name ++ "'. Remove $/${} for that, or use ${var?} to quiet." @@ -709,21 +708,20 @@ checkReadExpansions = CommandCheck (Exactly "read") check check cmd = mapM_ warning $ getVars cmd warning t = sequence_ $ do - var <- getSingleUnmodifiedVariable t - let name = bracedString var + name <- getSingleUnmodifiedBracedString t guard $ isVariableName name -- e.g. not $1 return . warn (getId t) 2229 $ "This does not read '" ++ name ++ "'. Remove $/${} for that, or use ${var?} to quiet." -- Return the single variable expansion that makes up this word, if any. -- e.g. $foo -> $foo, "$foo"'' -> $foo , "hello $name" -> Nothing -getSingleUnmodifiedVariable :: Token -> Maybe Token -getSingleUnmodifiedVariable word = +getSingleUnmodifiedBracedString :: Token -> Maybe String +getSingleUnmodifiedBracedString word = case getWordParts word of [t@(T_DollarBraced {})] -> let contents = bracedString t name = getBracedReference contents - in guard (contents == name) >> return t + in guard (contents == name) >> return (bracedString t) _ -> Nothing prop_checkAliasesUsesArgs1 = verify checkAliasesUsesArgs "alias a='cp $1 /a'" From 999b7e259683ed63c2102e82dfe9d548f1564cbc Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 17:20:49 -0400 Subject: [PATCH 351/763] Get rid of bracedString everywhere it's easy to --- src/ShellCheck/ASTLib.hs | 8 ++++---- src/ShellCheck/Analytics.hs | 24 ++++++++++++------------ src/ShellCheck/AnalyzerLib.hs | 8 ++++---- src/ShellCheck/Checks/Commands.hs | 6 +++--- src/ShellCheck/Checks/ShellSupport.hs | 8 ++++---- 5 files changed, 27 insertions(+), 27 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 2b40705..4f49243 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -139,8 +139,8 @@ bracedString (T_DollarBraced _ _ l) = concat $ oversimplify l bracedString _ = error "Internal shellcheck error, please report! (bracedString on non-variable)" -- Is this an expansion of multiple items of an array? -isArrayExpansion t@(T_DollarBraced _ _ _) = - let string = bracedString t in +isArrayExpansion (T_DollarBraced _ _ l) = + let string = concat $ oversimplify l in "@" `isPrefixOf` string || not ("#" `isPrefixOf` string) && "[@]" `isInfixOf` string isArrayExpansion _ = False @@ -148,8 +148,8 @@ isArrayExpansion _ = False -- Is it possible that this arg becomes multiple args? mayBecomeMultipleArgs t = willBecomeMultipleArgs t || f t where - f t@(T_DollarBraced _ _ _) = - let string = bracedString t in + f (T_DollarBraced _ _ l) = + let string = concat $ oversimplify l in "!" `isPrefixOf` string f (T_DoubleQuoted _ parts) = any f parts f (T_NormalWord _ parts) = any f parts diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index dd4e4c7..23852b1 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -781,8 +781,8 @@ checkShorthandIf _ _ = return () prop_checkDollarStar = verify checkDollarStar "for f in $*; do ..; done" prop_checkDollarStar2 = verifyNot checkDollarStar "a=$*" prop_checkDollarStar3 = verifyNot checkDollarStar "[[ $* = 'a b' ]]" -checkDollarStar p t@(T_NormalWord _ [b@(T_DollarBraced id _ _)]) - | bracedString b == "*" && +checkDollarStar p t@(T_NormalWord _ [T_DollarBraced id _ l]) + | concat (oversimplify l) == "*" && not (isStrictlyQuoteFree (parentMap p) t) = warn id 2048 "Use \"$@\" (with quotes) to prevent whitespace problems." checkDollarStar _ _ = return () @@ -1309,8 +1309,8 @@ prop_checkArithmeticDeref13= verifyNot checkArithmeticDeref "(( $$ ))" prop_checkArithmeticDeref14= verifyNot checkArithmeticDeref "(( $! ))" prop_checkArithmeticDeref15= verifyNot checkArithmeticDeref "(( ${!var} ))" prop_checkArithmeticDeref16= verifyNot checkArithmeticDeref "(( ${x+1} + ${x=42} ))" -checkArithmeticDeref params t@(TA_Expansion _ [b@(T_DollarBraced id _ _)]) = - unless (isException $ bracedString b) getWarning +checkArithmeticDeref params t@(TA_Expansion _ [T_DollarBraced id _ l]) = + unless (isException $ concat $ oversimplify l) getWarning where isException [] = True isException s@(h:_) = any (`elem` "/.:#%?*@$-!+=^,") s || isDigit h @@ -1940,7 +1940,7 @@ checkSpacefulness' onFind params t = T_DollarArithmetic _ _ -> SpaceNone T_Literal _ s -> fromLiteral s T_SingleQuoted _ s -> fromLiteral s - T_DollarBraced _ _ _ -> spacefulF $ getBracedReference $ bracedString x + T_DollarBraced _ _ l -> spacefulF $ getBracedReference $ concat $ oversimplify l T_NormalWord _ w -> isSpacefulWord spacefulF w T_DoubleQuoted _ w -> isSpacefulWord spacefulF w _ -> SpaceEmpty @@ -1955,13 +1955,13 @@ prop_CheckVariableBraces1 = verify checkVariableBraces "a='123'; echo $a" prop_CheckVariableBraces2 = verifyNot checkVariableBraces "a='123'; echo ${a}" prop_CheckVariableBraces3 = verifyNot checkVariableBraces "#shellcheck disable=SC2016\necho '$a'" prop_CheckVariableBraces4 = verifyNot checkVariableBraces "echo $* $1" -checkVariableBraces params t@(T_DollarBraced id False _) +checkVariableBraces params t@(T_DollarBraced id False l) | name `notElem` unbracedVariables = styleWithFix id 2250 "Prefer putting braces around variable references even when not strictly required." (fixFor t) where - name = getBracedReference $ bracedString t + name = getBracedReference $ concat $ oversimplify l fixFor token = fixWith [replaceStart (getId token) params 1 "${" ,replaceEnd (getId token) params 0 "}"] checkVariableBraces _ _ = return () @@ -2010,7 +2010,7 @@ checkQuotesInLiterals params t = squashesQuotes t = case t of - T_DollarBraced id _ _ -> "#" `isPrefixOf` bracedString t + T_DollarBraced id _ l -> "#" `isPrefixOf` concat (oversimplify l) _ -> False readF _ expr name = do @@ -2271,7 +2271,7 @@ checkUnassignedReferences' includeGlobals params t = warnings isInArray var t = any isArray $ getPath (parentMap params) t where isArray T_Array {} = True - isArray b@(T_DollarBraced _ _ _) | var /= getBracedReference (bracedString b) = True + isArray (T_DollarBraced _ _ l) | var /= getBracedReference (concat $ oversimplify l) = True isArray _ = False isGuarded (T_DollarBraced _ _ v) = @@ -2399,7 +2399,7 @@ prop_checkPrefixAssign2 = verifyNot checkPrefixAssignmentReference "var=$(echo $ checkPrefixAssignmentReference params t@(T_DollarBraced id _ value) = check path where - name = getBracedReference $ bracedString t + name = getBracedReference $ concat $ oversimplify value path = getPath (parentMap params) t idPath = map getId path @@ -3032,7 +3032,7 @@ checkReturnAgainstZero _ token = isZero t = getLiteralString t == Just "0" isExitCode t = case getWordParts t of - [exp@T_DollarBraced {}] -> bracedString exp == "?" + [T_DollarBraced _ _ l] -> concat (oversimplify l) == "?" _ -> False message id = style id 2181 "Check exit code directly with e.g. 'if mycmd;', not indirectly with $?." @@ -3223,7 +3223,7 @@ checkSplittingInArrays params t = T_DollarBraced id _ str | not (isCountingReference part) && not (isQuotedAlternativeReference part) - && getBracedReference (bracedString part) `notElem` variablesWithoutSpaces + && getBracedReference (concat $ oversimplify str) `notElem` variablesWithoutSpaces -> warn id 2206 $ if shellType params == Ksh then "Quote to prevent word splitting/globbing, or split robustly with read -A or while read." diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 3a7f7da..69ebfe6 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -498,7 +498,7 @@ getModifiedVariables t = return (t, token, str, DataString SourceChecked) T_DollarBraced _ _ l -> do - let string = bracedString t + let string = concat $ oversimplify l let modifier = getBracedModifier string guard $ any (`isPrefixOf` modifier) ["=", ":="] return (t, t, getBracedReference string, DataString $ SourceFrom [l]) @@ -703,7 +703,7 @@ getOffsetReferences mods = fromMaybe [] $ do getReferencedVariables parents t = case t of - T_DollarBraced id _ l -> let str = bracedString t in + T_DollarBraced id _ l -> let str = concat $ oversimplify l in (t, t, getBracedReference str) : map (\x -> (l, l, x)) ( getIndexReferences str @@ -895,8 +895,8 @@ isCountingReference _ = False -- FIXME: doesn't handle ${a:+$var} vs ${a:+"$var"} isQuotedAlternativeReference t = case t of - T_DollarBraced _ _ _ -> - getBracedModifier (bracedString t) `matches` re + T_DollarBraced _ _ l -> + getBracedModifier (concat $ oversimplify l) `matches` re _ -> False where re = mkRegex "(^|\\]):?\\+" diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 4e9c045..3ac14c3 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -718,10 +718,10 @@ checkReadExpansions = CommandCheck (Exactly "read") check getSingleUnmodifiedBracedString :: Token -> Maybe String getSingleUnmodifiedBracedString word = case getWordParts word of - [t@(T_DollarBraced {})] -> - let contents = bracedString t + [T_DollarBraced _ _ l] -> + let contents = concat $ oversimplify l name = getBracedReference contents - in guard (contents == name) >> return (bracedString t) + in guard (contents == name) >> return contents _ -> Nothing prop_checkAliasesUsesArgs1 = verify checkAliasesUsesArgs "alias a='cp $1 /a'" diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 49d3212..d9deff6 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -243,7 +243,7 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do when (isBashVariable var) $ warnMsg id $ var ++ " is" where - str = bracedString t + str = concat $ oversimplify token var = getBracedReference str check (regex, feature) = when (isJust $ matchRegex regex str) $ warnMsg id feature @@ -506,13 +506,13 @@ checkMultiDimensionalArrays = ForShell [Bash] f case token of T_Assignment _ _ name (first:second:_) _ -> about second T_IndexedElement _ (first:second:_) _ -> about second - T_DollarBraced {} -> - when (isMultiDim token) $ about token + T_DollarBraced _ _ l -> + when (isMultiDim l) $ about token _ -> return () about t = warn (getId t) 2180 "Bash does not support multidimensional arrays. Use 1D or associative arrays." re = mkRegex "^\\[.*\\]\\[.*\\]" -- Fixme, this matches ${foo:- [][]} and such as well - isMultiDim t = getBracedModifier (bracedString t) `matches` re + isMultiDim l = getBracedModifier (concat $ oversimplify l) `matches` re prop_checkPS11 = verify checkPS1Assignments "PS1='\\033[1;35m\\$ '" prop_checkPS11a= verify checkPS1Assignments "export PS1='\\033[1;35m\\$ '" From b58bb4ba9d338d1b2ce8cf690deeb6ac43ce97a1 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 17:26:24 -0400 Subject: [PATCH 352/763] Move bracedString to be local to its last use site --- src/ShellCheck/ASTLib.hs | 4 ---- src/ShellCheck/Analytics.hs | 4 ++++ 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 4f49243..efa8cdb 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -134,10 +134,6 @@ isUnquotedFlag token = fromMaybe False $ do str <- getLeadingUnquotedString token return $ "-" `isPrefixOf` str --- Given a T_DollarBraced, return a simplified version of the string contents. -bracedString (T_DollarBraced _ _ l) = concat $ oversimplify l -bracedString _ = error "Internal shellcheck error, please report! (bracedString on non-variable)" - -- Is this an expansion of multiple items of an array? isArrayExpansion (T_DollarBraced _ _ l) = let string = concat $ oversimplify l in diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 23852b1..dd98370 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1869,6 +1869,10 @@ checkSpacefulness params = checkSpacefulness' onFind params any (`isPrefixOf` modifier) ["=", ":="] && isParamTo parents ":" token + -- Given a T_DollarBraced, return a simplified version of the string contents. + bracedString (T_DollarBraced _ _ l) = concat $ oversimplify l + bracedString _ = error "Internal shellcheck error, please report! (bracedString on non-variable)" + prop_checkSpacefulness4v= verifyTree checkVerboseSpacefulness "foo=3; foo=$(echo $foo)" prop_checkSpacefulness8v= verifyTree checkVerboseSpacefulness "a=foo\\ bar; a=foo; rm $a" prop_checkSpacefulness28v = verifyTree checkVerboseSpacefulness "exec {n}>&1; echo $n" From 163c710ba7da07334999bf18804c971a2be41368 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 12 Apr 2020 16:15:45 -0400 Subject: [PATCH 353/763] Clean up and optimize getSuspiciousRegexWildcard --- src/ShellCheck/Checks/Commands.hs | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index c68bdd1..395ed2b 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -283,17 +283,11 @@ checkGrepRe = CommandCheck (Basename "grep") check where candidates = sampleWords ++ map (\(x:r) -> toUpper x : r) sampleWords - getSuspiciousRegexWildcard str = - if not $ str `matches` contra - then do - match <- matchRegex suspicious str - str <- match !!! 0 - str !!! 0 - else - fail "looks good" - where - suspicious = mkRegex "([A-Za-z1-9])\\*" - contra = mkRegex "[^a-zA-Z1-9]\\*|[][^$+\\\\]" + getSuspiciousRegexWildcard str = case matchRegex suspicious str of + Just [[c]] | not (str `matches` contra) -> Just c + _ -> fail "looks good" + suspicious = mkRegex "([A-Za-z1-9])\\*" + contra = mkRegex "[^a-zA-Z1-9]\\*|[][^$+\\\\]" prop_checkTrapQuotes1 = verify checkTrapQuotes "trap \"echo $num\" INT" From e0daa936d2b82f7a5993047bb95f9eb3eaa4db01 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Fri, 24 Apr 2020 22:14:08 -0400 Subject: [PATCH 354/763] Revert "Use fromRight instead of reimplementing it" We still support GHC 8.0, which didn't have fromRight. This reverts commit 64c31d91422e3c48cc7ed0176b069fa11456cee8. --- src/ShellCheck/Formatter/GCC.hs | 3 +-- src/ShellCheck/Formatter/JSON1.hs | 3 +-- src/ShellCheck/Formatter/TTY.hs | 3 +-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/ShellCheck/Formatter/GCC.hs b/src/ShellCheck/Formatter/GCC.hs index a7307a9..9c5fa5f 100644 --- a/src/ShellCheck/Formatter/GCC.hs +++ b/src/ShellCheck/Formatter/GCC.hs @@ -22,7 +22,6 @@ module ShellCheck.Formatter.GCC (format) where import ShellCheck.Interface import ShellCheck.Formatter.Format -import Data.Either import Data.List import GHC.Exts import System.IO @@ -45,7 +44,7 @@ outputAll cr sys = mapM_ f groups f group = do let filename = sourceFile (head group) result <- (siReadFile sys) filename - let contents = fromRight "" result + let contents = either (const "") id result outputResult filename contents group outputResult filename contents warnings = do diff --git a/src/ShellCheck/Formatter/JSON1.hs b/src/ShellCheck/Formatter/JSON1.hs index edc1974..7335d8c 100644 --- a/src/ShellCheck/Formatter/JSON1.hs +++ b/src/ShellCheck/Formatter/JSON1.hs @@ -24,7 +24,6 @@ import ShellCheck.Interface import ShellCheck.Formatter.Format import Data.Aeson -import Data.Either import Data.IORef import Data.Monoid import GHC.Exts @@ -119,7 +118,7 @@ collectResult ref cr sys = mapM_ f groups f group = do let filename = sourceFile (head group) result <- siReadFile sys filename - let contents = fromRight "" result + let contents = either (const "") id result let comments' = makeNonVirtual comments contents modifyIORef ref (\x -> comments' ++ x) diff --git a/src/ShellCheck/Formatter/TTY.hs b/src/ShellCheck/Formatter/TTY.hs index 44bda1e..0d474d7 100644 --- a/src/ShellCheck/Formatter/TTY.hs +++ b/src/ShellCheck/Formatter/TTY.hs @@ -25,7 +25,6 @@ import ShellCheck.Formatter.Format import Control.Monad import Data.Array -import Data.Either import Data.Foldable import Data.Ord import Data.IORef @@ -123,7 +122,7 @@ outputResult options ref result sys = do outputForFile color sys comments = do let fileName = sourceFile (head comments) result <- (siReadFile sys) fileName - let contents = fromRight "" result + let contents = either (const "") id result let fileLinesList = lines contents let lineCount = length fileLinesList let fileLines = listArray (1, lineCount) fileLinesList From 60e80e4ce1323b4f5944018dd5e132b0287cffd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Skytt=C3=A4?= Date: Sat, 25 Apr 2020 08:29:38 +0300 Subject: [PATCH 355/763] Spelling fixes --- src/ShellCheck/Parser.hs | 2 +- test/check_release | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index b96aee7..46d7ace 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -2622,7 +2622,7 @@ readFunctionDefinition = called "function" $ do readWithoutFunction = try $ do name <- many1 functionChars - guard $ name /= "time" -- Interfers with time ( foo ) + guard $ name /= "time" -- Interferes with time ( foo ) spacing readParens return $ \id -> T_Function id (FunctionKeyword False) (FunctionParentheses True) name diff --git a/test/check_release b/test/check_release index b9cd34b..91001fb 100755 --- a/test/check_release +++ b/test/check_release @@ -9,7 +9,7 @@ fail() { if git diff | grep -q "" then - fail "There are uncommited changes" + fail "There are uncommitted changes" fi current=$(git tag --points-at) From 1d126960f342f8cdda7fcc5e083284c3196e2fa2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Skytt=C3=A4?= Date: Sat, 25 Apr 2020 08:33:10 +0300 Subject: [PATCH 356/763] Use SC prefix for disable= in man page --- shellcheck.1.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shellcheck.1.md b/shellcheck.1.md index 50eaddc..6d2a732 100644 --- a/shellcheck.1.md +++ b/shellcheck.1.md @@ -254,7 +254,7 @@ Valid keys are: **shell** : Overrides the shell detected from the shebang. This is useful for files meant to be included (and thus lacking a shebang), or possibly - as a more targeted alternative to 'disable=2039'. + as a more targeted alternative to 'disable=SC2039'. # RC FILES From 2030b83607a84a84a96bbc73d8c541fdaa950fa5 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 3 May 2020 11:29:00 -0700 Subject: [PATCH 357/763] Warn about duplicate uses of stdin/out/err --- CHANGELOG.md | 3 + src/ShellCheck/Analytics.hs | 160 +++++++++++++++++++++++++++++++----- 2 files changed, 142 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 181cc54..fbc2ab7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,7 @@ ## Git +### Added +- SC2259/SC2260: Warn when redirections override pipes +- SC2261: Warn about multiple competing redirections ## v0.7.1 - 2020-04-04 ### Fixed diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 125835a..4281ace 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -3282,25 +3282,92 @@ prop_checkPipeToNowhere6 = verifyNot checkPipeToNowhere "ls | echo $(cat)" prop_checkPipeToNowhere7 = verifyNot checkPipeToNowhere "echo foo | var=$(cat) ls" prop_checkPipeToNowhere8 = verify checkPipeToNowhere "foo | true" prop_checkPipeToNowhere9 = verifyNot checkPipeToNowhere "mv -i f . < /dev/stdin" +prop_checkPipeToNowhere10 = verify checkPipeToNowhere "ls > file | grep foo" +prop_checkPipeToNowhere11 = verify checkPipeToNowhere "ls | grep foo < file" +prop_checkPipeToNowhere12 = verify checkPipeToNowhere "ls > foo > bar" +prop_checkPipeToNowhere13 = verify checkPipeToNowhere "ls > foo 2> bar > baz" +prop_checkPipeToNowhere14 = verify checkPipeToNowhere "ls > foo &> bar" +prop_checkPipeToNowhere15 = verifyNot checkPipeToNowhere "ls > foo 2> bar |& grep 'No space left'" +prop_checkPipeToNowhere16 = verifyNot checkPipeToNowhere "echo World | cat << EOF\nhello $(cat)\nEOF\n" +prop_checkPipeToNowhere17 = verify checkPipeToNowhere "echo World | cat << 'EOF'\nhello $(cat)\nEOF\n" +prop_checkPipeToNowhere18 = verifyNot checkPipeToNowhere "ls 1>&3 3>&1 3>&- | wc -l" + +data PipeType = StdoutPipe | StdoutStderrPipe | NoPipe deriving (Eq) checkPipeToNowhere :: Parameters -> Token -> WriterT [TokenComment] Identity () -checkPipeToNowhere _ t = +checkPipeToNowhere params t = case t of - T_Pipeline _ _ (first:rest) -> mapM_ checkPipe rest + T_Pipeline _ pipes cmds -> + mapM_ checkPipe $ commandsWithContext pipes cmds T_Redirecting _ redirects cmd | any redirectsStdin redirects -> checkRedir cmd _ -> return () where - checkPipe redir = sequence_ $ do - cmd <- getCommand redir - name <- getCommandBasename cmd - guard $ name `elem` nonReadingCommands - guard . not $ hasAdditionalConsumers cmd - -- Confusing echo for cat is so common that it's worth a special case - let suggestion = - if name == "echo" - then "Did you want 'cat' instead?" - else "Wrong command or missing xargs?" - return $ warn (getId cmd) 2216 $ - "Piping to '" ++ name ++ "', a command that doesn't read stdin. " ++ suggestion + checkPipe (input, stage, output) = do + let hasConsumers = hasAdditionalConsumers stage + let hasProducers = hasAdditionalProducers stage + + sequence_ $ do + cmd <- getCommand stage + name <- getCommandBasename cmd + guard $ name `elem` nonReadingCommands + guard $ not hasConsumers && input /= NoPipe + + -- Confusing echo for cat is so common that it's worth a special case + let suggestion = + if name == "echo" + then "Did you want 'cat' instead?" + else "Wrong command or missing xargs?" + return $ warn (getId cmd) 2216 $ + "Piping to '" ++ name ++ "', a command that doesn't read stdin. " ++ suggestion + + sequence_ $ do + T_Redirecting _ redirs cmd <- return stage + fds <- sequence $ map getRedirectionFds redirs + + let fdAndToken :: [(Integer, Token)] + fdAndToken = + concatMap (\(list, redir) -> map (\n -> (n, redir)) list) $ + zip fds redirs + + let fdMap = + Map.fromListWith (++) $ + map (\(a,b) -> (a,[b])) fdAndToken + + let inputWarning = sequence_ $ do + guard $ input /= NoPipe && not hasConsumers + (override:_) <- Map.lookup 0 fdMap + return $ err (getOpId override) 2259 $ + "This redirection overrides piped input. To use both, merge or pass filename." + + -- Only produce output warnings for regular pipes, since these are + -- way more common, and `foo > out 2> err |& foo` can still write + -- to stderr if the files fail to open + let outputWarning = sequence_ $ do + guard $ output == StdoutPipe && not hasProducers + (override:_) <- Map.lookup 1 fdMap + return $ err (getOpId override) 2260 $ + "This redirection overrides the output pipe. Use 'tee' to output to both." + + return $ do + inputWarning + outputWarning + mapM_ warnAboutDupes $ Map.assocs fdMap + + warnAboutDupes (n, list@(_:_:_)) = + forM_ list $ \c -> err (getOpId c) 2261 $ + "Multiple redirections compete for " ++ str n ++ ". Combine, or use " ++ alternative ++ "." + warnAboutDupes _ = return () + + alternative = + if shellType params `elem` [Bash, Ksh] + then "process substitutions or temp files" + else "temporary files" + + str n = + case n of + 0 -> "stdin" + 1 -> "stdout" + 2 -> "stderr" + _ -> "FD " ++ show n checkRedir cmd = sequence_ $ do name <- getCommandBasename cmd @@ -3315,23 +3382,74 @@ checkPipeToNowhere _ t = "Redirecting to '" ++ name ++ "', a command that doesn't read stdin. " ++ suggestion -- Could any words in a SimpleCommand consume stdin (e.g. echo "$(cat)")? - hasAdditionalConsumers t = isNothing $ - doAnalysis (guard . not . mayConsume) t + hasAdditionalConsumers = treeContains mayConsume + -- Could any words in a SimpleCommand produce stdout? E.g. >(tee foo) + hasAdditionalProducers = treeContains mayProduce + treeContains pred t = isNothing $ + doAnalysis (guard . not . pred) t mayConsume t = case t of - T_ProcSub {} -> True + T_ProcSub _ "<" _ -> True T_Backticked {} -> True T_DollarExpansion {} -> True _ -> False - redirectsStdin t = + mayProduce t = case t of - T_FdRedirect _ _ (T_IoFile _ T_Less {} _) -> True - T_FdRedirect _ _ T_HereDoc {} -> True - T_FdRedirect _ _ T_HereString {} -> True + T_ProcSub _ ">" _ -> True _ -> False + getOpId t = + case t of + T_FdRedirect _ _ x -> getOpId x + T_IoFile _ op _ -> getId op + _ -> getId t + + getRedirectionFds t = + case t of + T_FdRedirect _ "" x -> getDefaultFds x + T_FdRedirect _ "&" _ -> return [1, 2] + T_FdRedirect _ num x | all isDigit num -> + -- Don't report the number unless we know what it is. + -- This avoids triggering on 3>&1 1>&3 + getDefaultFds x *> return [read num] + -- Don't bother with {fd}>42 and such + _ -> Nothing + + getDefaultFds redir = + case redir of + T_HereDoc {} -> return [0] + T_HereString {} -> return [0] + T_IoFile _ op _ -> + case op of + T_Less {} -> return [0] + T_Greater {} -> return [1] + T_DGREAT {} -> return [1] + T_GREATAND {} -> return [1, 2] + T_CLOBBER {} -> return [1] + T_IoDuplicate _ op "-" -> getDefaultFds op + _ -> Nothing + _ -> Nothing + + redirectsStdin t = + fromMaybe False $ do + fds <- getRedirectionFds t + return $ 0 `elem` fds + + pipeType t = + case t of + T_Pipe _ "|" -> StdoutPipe + T_Pipe _ "|&" -> StdoutStderrPipe + _ -> NoPipe + + commandsWithContext pipes cmds = + let pipeTypes = map pipeType pipes + inputs = NoPipe : pipeTypes + outputs = pipeTypes ++ [NoPipe] + in + zip3 inputs cmds outputs + prop_checkUseBeforeDefinition1 = verifyTree checkUseBeforeDefinition "f; f() { true; }" prop_checkUseBeforeDefinition2 = verifyNotTree checkUseBeforeDefinition "f() { true; }; f" prop_checkUseBeforeDefinition3 = verifyNotTree checkUseBeforeDefinition "if ! mycmd --version; then mycmd() { true; }; fi" From d6adbfde787a46e2d61dc96ffdf8c9a6f800390e Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 3 May 2020 21:46:16 -0700 Subject: [PATCH 358/763] Improve SC2259/60/61 messages --- src/ShellCheck/Analytics.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 4281ace..9c9a86d 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -3336,7 +3336,7 @@ checkPipeToNowhere params t = guard $ input /= NoPipe && not hasConsumers (override:_) <- Map.lookup 0 fdMap return $ err (getOpId override) 2259 $ - "This redirection overrides piped input. To use both, merge or pass filename." + "This redirection overrides piped input. To use both, merge or pass filenames." -- Only produce output warnings for regular pipes, since these are -- way more common, and `foo > out 2> err |& foo` can still write @@ -3354,7 +3354,7 @@ checkPipeToNowhere params t = warnAboutDupes (n, list@(_:_:_)) = forM_ list $ \c -> err (getOpId c) 2261 $ - "Multiple redirections compete for " ++ str n ++ ". Combine, or use " ++ alternative ++ "." + "Multiple redirections compete for " ++ str n ++ ". Use cat, tee, or pass filenames instead." warnAboutDupes _ = return () alternative = From c2a15ce8e906ae6a12bcbe32eac7ac586fbcb59b Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 3 May 2020 21:52:25 -0700 Subject: [PATCH 359/763] Allow disabling SC1072/SC1073 with annotations (fixes #1931) --- CHANGELOG.md | 5 +++++ src/ShellCheck/Checker.hs | 7 +++++++ src/ShellCheck/Parser.hs | 19 +++++++++++++++---- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fbc2ab7..6089c76 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,11 @@ - SC2259/SC2260: Warn when redirections override pipes - SC2261: Warn about multiple competing redirections +### Fixed +- SC1072/SC1073 now respond to disable annotations, though ignoring parse errors + is still purely cosmetic and does not allow ShellCheck to continue. + + ## v0.7.1 - 2020-04-04 ### Fixed - `-f diff` no longer claims that it found more issues when it didn't diff --git a/src/ShellCheck/Checker.hs b/src/ShellCheck/Checker.hs index 120f7d8..f639ab3 100644 --- a/src/ShellCheck/Checker.hs +++ b/src/ShellCheck/Checker.hs @@ -295,6 +295,13 @@ prop_canDisableShebangWarning = null $ result csScript = "#shellcheck disable=SC2148\nfoo" } +prop_canDisableParseErrors = null $ result + where + result = checkWithSpec [] emptyCheckSpec { + csFilename = "file.sh", + csScript = "#shellcheck disable=SC1073,SC1072,SC2148\n()" + } + prop_shExtensionDoesntMatter = result == [2148] where result = checkWithSpec [] emptyCheckSpec { diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 46d7ace..7eb2c32 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -253,7 +253,11 @@ ignoreProblemsOf p = do shouldIgnoreCode code = do context <- getCurrentContexts checkSourced <- Mr.asks checkSourced - return $ any (disabling checkSourced) context + return $ any (contextItemDisablesCode checkSourced code) context + +-- Does this item on the context stack disable warnings for 'code'? +contextItemDisablesCode :: Bool -> Integer -> Context -> Bool +contextItemDisablesCode alsoCheckSourced code = disabling alsoCheckSourced where disabling checkSourced item = case item of @@ -263,6 +267,8 @@ shouldIgnoreCode code = do disabling' (DisableComment n) = code == n disabling' _ = False + + getCurrentAnnotations includeSource = concatMap get . takeWhile (not . isBoundary) <$> getCurrentContexts where @@ -3313,16 +3319,21 @@ parseShell env name contents = do prRoot = Just $ reattachHereDocs script (hereDocMap userstate) } - Left err -> + Left err -> do + let context = contextStack state return newParseResult { prComments = map toPositionedComment $ - notesForContext (contextStack state) - ++ [makeErrorFor err] + (filter (not . isIgnored context) $ + notesForContext context + ++ [makeErrorFor err]) ++ parseProblems state, prTokenPositions = Map.empty, prRoot = Nothing } + where + -- A final pass for ignoring parse errors after failed parsing + isIgnored stack note = any (contextItemDisablesCode False (codeForParseNote note)) stack notesForContext list = zipWith ($) [first, second] $ filter isName list where From 536cb584f486fbd93feb2c3fe8f30703e52a6118 Mon Sep 17 00:00:00 2001 From: geeseven <2334728+geeseven@users.noreply.github.com> Date: Tue, 12 May 2020 14:58:15 -0500 Subject: [PATCH 360/763] update dependency free AUR package The `shellcheck-static` AUR package has been removed and `shellcheck-bin` is an option for a dependency free AUR package. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ce57d07..bfb3a99 100644 --- a/README.md +++ b/README.md @@ -148,7 +148,7 @@ On Arch Linux based distros: pacman -S shellcheck -or get the dependency free [shellcheck-static](https://aur.archlinux.org/packages/shellcheck-static/) from the AUR. +or get the dependency free [shellcheck-bin](https://aur.archlinux.org/packages/shellcheck-bin/) from the AUR. On Gentoo based distros: From a08ad3bee91179009a2daa34427148b03c885af6 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Fri, 8 May 2020 10:19:33 -0700 Subject: [PATCH 361/763] Count $# as an argument reference in SC2120 --- src/ShellCheck/Analytics.hs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 9c9a86d..fcc942b 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2555,6 +2555,7 @@ prop_checkUnpassedInFunctions10= verifyNotTree checkUnpassedInFunctions "foo() { prop_checkUnpassedInFunctions11= verifyNotTree checkUnpassedInFunctions "foo() { bar() { echo $1; }; bar baz; }; foo;" prop_checkUnpassedInFunctions12= verifyNotTree checkUnpassedInFunctions "foo() { echo ${!var*}; }; foo;" prop_checkUnpassedInFunctions13= verifyNotTree checkUnpassedInFunctions "# shellcheck disable=SC2120\nfoo() { echo $1; }\nfoo\n" +prop_checkUnpassedInFunctions14= verifyTree checkUnpassedInFunctions "foo() { echo $#; }; foo" checkUnpassedInFunctions params root = execWriter $ mapM_ warnForGroup referenceGroups where @@ -2596,7 +2597,7 @@ checkUnpassedInFunctions params root = return $ tell [(str, null args, t)] checkCommand _ = Nothing - isPositional str = str == "*" || str == "@" + isPositional str = str == "*" || str == "@" || str == "#" || (all isDigit str && str /= "0" && str /= "") isArgumentless (_, b, _) = b From 5cf2c00ff7d4d823b5cbe6420b2a48f1ed0447b3 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 25 May 2020 23:21:47 -0700 Subject: [PATCH 362/763] Warn about defining and using an alias in a single command (fixes #1807) --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 72 +++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6089c76..deacf5b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ### Added - SC2259/SC2260: Warn when redirections override pipes - SC2261: Warn about multiple competing redirections +- SC2262/SC2263: Warn about aliases declared and used in the same parsing unit ### Fixed - SC1072/SC1073 now respond to disable annotations, though ignoring parse errors diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index fcc942b..977d415 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -64,6 +64,7 @@ treeChecks = [ ,checkUncheckedCdPushdPopd ,checkArrayAssignmentIndices ,checkUseBeforeDefinition + ,checkAliasUsedInSameParsingUnit ] runAnalytics :: AnalysisSpec -> [TokenComment] @@ -3740,6 +3741,77 @@ checkModifiedArithmeticInRedirection params t = unless (shellType params == Dash warnFor id = warn id 2257 "Arithmetic modifications in command redirections may be discarded. Do them separately." +prop_checkAliasUsedInSameParsingUnit1 = verifyTree checkAliasUsedInSameParsingUnit "alias x=y; x" +prop_checkAliasUsedInSameParsingUnit2 = verifyNotTree checkAliasUsedInSameParsingUnit "alias x=y\nx" +prop_checkAliasUsedInSameParsingUnit3 = verifyTree checkAliasUsedInSameParsingUnit "{ alias x=y\nx\n}" +prop_checkAliasUsedInSameParsingUnit4 = verifyNotTree checkAliasUsedInSameParsingUnit "alias x=y; 'x';" +prop_checkAliasUsedInSameParsingUnit5 = verifyNotTree checkAliasUsedInSameParsingUnit ":\n{\n#shellcheck disable=SC2262\nalias x=y\nx\n}" +prop_checkAliasUsedInSameParsingUnit6 = verifyNotTree checkAliasUsedInSameParsingUnit ":\n{\n#shellcheck disable=SC2262\nalias x=y\nalias x=z\nx\n}" -- Only consider the first instance +checkAliasUsedInSameParsingUnit :: Parameters -> Token -> [TokenComment] +checkAliasUsedInSameParsingUnit params root = + let + -- Get all root commands + commands = concat $ getCommandSequences root + -- Group them by whether they start on the same line where the previous one ended + units = groupByLink followsOnLine commands + in + execWriter $ sequence_ $ map checkUnit units + where + lineSpan t = + let m = tokenPositions params in do + (start, end) <- Map.lookup t m + return $ (posLine start, posLine end) + + followsOnLine a b = fromMaybe False $ do + (_, end) <- lineSpan (getId a) + (start, _) <- lineSpan (getId b) + return $ end == start + + checkUnit :: [Token] -> Writer [TokenComment] () + checkUnit unit = evalStateT (mapM_ (doAnalysis findCommands) unit) (Map.empty) + + isSourced t = + let + f (T_SourceCommand {}) = True + f _ = False + in + any f $ getPath (parentMap params) t + + findCommands :: Token -> StateT (Map.Map String Token) (Writer [TokenComment]) () + findCommands t = case t of + T_SimpleCommand _ _ (cmd:args) -> + case getUnquotedLiteral cmd of + Just "alias" -> + mapM_ addAlias args + Just name | '/' `notElem` name -> do + cmd <- gets (Map.lookup name) + case cmd of + Just alias -> + unless (isSourced t || shouldIgnoreCode params 2262 alias) $ do + warn (getId alias) 2262 "This alias can't be defined and used in the same parsing unit. Use a function instead." + info (getId t) 2263 "Since they're in the same parsing unit, this command will not refer to the previously mentioned alias." + _ -> return () + _ -> return () + _ -> return () + addAlias arg = do + let (name, value) = break (== '=') $ getLiteralStringDef "-" arg + when (isVariableName name && not (null value)) $ + modify (Map.insertWith (\new old -> old) name arg) + +-- Like groupBy, but compares pairs of adjacent elements, rather than against the first of the span +prop_groupByLink1 = groupByLink (\a b -> a+1 == b) [1,2,3,2,3,7,8,9] == [[1,2,3], [2,3], [7,8,9]] +prop_groupByLink2 = groupByLink (==) ([] :: [()]) == [] +groupByLink :: (a -> a -> Bool) -> [a] -> [[a]] +groupByLink f list = + case list of + [] -> [] + (x:xs) -> g x [] xs + where + g current span (next:rest) = + if f current next + then g next (current:span) rest + else (reverse $ current:span) : g next [] rest + g current span [] = [reverse (current:span)] return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) From a2b5b6a50030d50da832e9f5c314d48e4746034f Mon Sep 17 00:00:00 2001 From: Onno Zweers Date: Wed, 3 Jun 2020 11:37:53 +0200 Subject: [PATCH 363/763] Rephrase: *Shellcheck* can't follow non-constant source The message "Can't follow non-constant source. Use a directive to specify location." set me off on the wrong foot. At first, I thought it meant "In bash, you can't source a script specified by a variable." It was only after reading the wiki page for this message https://github.com/koalaman/shellcheck/wiki/SC1090 that I understood that the problem is that *shellcheck* can't check the sourced file. So I would suggest to rephrase this message so that it is more clear that the problem is in the checking, not in the running of the script. --- src/ShellCheck/Parser.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 7eb2c32..01795ed 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -2129,7 +2129,7 @@ readSource t@(T_Redirecting _ _ (T_SimpleCommand cmdId _ (cmd:file':rest'))) = d case literalFile of Nothing -> do parseNoteAtId (getId file) WarningC 1090 - "Can't follow non-constant source. Use a directive to specify location." + "Shellcheck can't follow non-constant source. Use a directive to specify location." return t Just filename -> do proceed <- shouldFollow filename From 12d9c1b76d0b25637552e693b832fdcedf90238c Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 24 Jun 2020 11:50:27 -0700 Subject: [PATCH 364/763] Clarify that SC1090 refers to ShellCheck, not sh --- src/ShellCheck/Parser.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 7eb2c32..e8e04e0 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -2129,7 +2129,7 @@ readSource t@(T_Redirecting _ _ (T_SimpleCommand cmdId _ (cmd:file':rest'))) = d case literalFile of Nothing -> do parseNoteAtId (getId file) WarningC 1090 - "Can't follow non-constant source. Use a directive to specify location." + "ShellCheck can't follow non-constant source. Use a directive to specify location." return t Just filename -> do proceed <- shouldFollow filename From a61d8a232c403d73c071719292683e9e7cc24763 Mon Sep 17 00:00:00 2001 From: Aurelio Jargas Date: Fri, 26 Jun 2020 02:13:33 +0200 Subject: [PATCH 365/763] SC1102: Fix typo in error message: substition --- src/ShellCheck/Parser.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index e8e04e0..1f6b45b 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -1557,7 +1557,7 @@ readDollarExpression = do readDollarExp = arithmetic <|> readDollarExpansion <|> readDollarBracket <|> readDollarBraceCommandExpansion <|> readDollarBraced <|> readDollarVariable where arithmetic = readAmbiguous "$((" readDollarArithmetic readDollarExpansion (\pos -> - parseNoteAt pos ErrorC 1102 "Shells disambiguate $(( differently or not at all. For $(command substition), add space after $( . For $((arithmetics)), fix parsing errors.") + parseNoteAt pos ErrorC 1102 "Shells disambiguate $(( differently or not at all. For $(command substitution), add space after $( . For $((arithmetics)), fix parsing errors.") prop_readDollarSingleQuote = isOk readDollarSingleQuote "$'foo\\\'lol'" readDollarSingleQuote = called "$'..' expression" $ do From 6b88a341f33efc1957f6d0621b2acf80270a9fbd Mon Sep 17 00:00:00 2001 From: Stavros Ntentos <133706+stdedos@users.noreply.github.com> Date: Sun, 28 Jun 2020 01:47:02 +0300 Subject: [PATCH 366/763] Autolink https://www.shellcheck.net/ --- .github/ISSUE_TEMPLATE.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index b7f88e8..44d151e 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -2,10 +2,10 @@ - Rule Id (if any, e.g. SC1000): - My shellcheck version (`shellcheck --version` or "online"): - [ ] The rule's wiki page does not already cover this (e.g. https://shellcheck.net/wiki/SC2086) -- [ ] I tried on shellcheck.net and verified that this is still a problem on the latest commit +- [ ] I tried on https://www.shellcheck.net/ and verified that this is still a problem on the latest commit #### For new checks and feature suggestions -- [ ] shellcheck.net (i.e. the latest commit) currently gives no useful warnings about this +- [ ] https://www.shellcheck.net/ (i.e. the latest commit) currently gives no useful warnings about this - [ ] I searched through https://github.com/koalaman/shellcheck/issues and didn't find anything related From 739eaadbf515f4548b5d29e3611d70bd56ab7a4c Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 28 Jun 2020 16:01:15 -0700 Subject: [PATCH 367/763] Warn about extra spaces between ((s in for((;;)) --- src/ShellCheck/Parser.hs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index e8e04e0..b3b3425 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -2485,19 +2485,31 @@ readForClause = called "for loop" $ do readArithmetic id <|> readRegular id where readArithmetic id = called "arithmetic for condition" $ do - try $ string "((" + readArithmeticDelimiter '(' "Missing second '(' to start arithmetic for ((;;)) loop" x <- readArithmeticContents char ';' >> spacing y <- readArithmeticContents char ';' >> spacing z <- readArithmeticContents spacing - string "))" + readArithmeticDelimiter ')' "Missing second ')' to terminate 'for ((;;))' loop condition" spacing optional $ readSequentialSep >> spacing group <- readBraced <|> readDoGroup id return $ T_ForArithmetic id x y z group + -- For c='(' read "((" and be lenient about spaces + readArithmeticDelimiter c msg = do + char c + startPos <- getPosition + sp <- spacing + endPos <- getPosition + char c <|> do + parseProblemAt startPos ErrorC 1137 msg + fail "" + unless (null sp) $ + parseProblemAtWithEnd startPos endPos ErrorC 1138 $ "Remove spaces between " ++ [c,c] ++ " in arithmetic for loop." + readBraced = do (T_BraceGroup _ list) <- readBraceGroup return list From 210cdcd01a91923a7e1d04b4392616460ec5be68 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 28 Jun 2020 17:24:07 -0700 Subject: [PATCH 368/763] Treat $x/ or $(x)/ as ./ when finding sourced files (fixes #1998) --- CHANGELOG.md | 3 +++ src/ShellCheck/ASTLib.hs | 5 +++++ src/ShellCheck/Checker.hs | 6 ++++++ src/ShellCheck/Parser.hs | 12 +++++++++++- 4 files changed, 25 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index deacf5b..fde8b0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ - SC1072/SC1073 now respond to disable annotations, though ignoring parse errors is still purely cosmetic and does not allow ShellCheck to continue. +### Changed +- SC1090: A leading `$x/` or `$(x)/` is now treated as `./` when locating files + ## v0.7.1 - 2020-04-04 ### Fixed diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 4fdf078..29ce27f 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -503,6 +503,11 @@ isCommandSubstitution t = case t of T_Backticked {} -> True _ -> False +-- Is this an expansion that results in a simple string? +isStringExpansion t = isCommandSubstitution t || case t of + T_DollarArithmetic {} -> True + T_DollarBraced {} -> not (isArrayExpansion t) + _ -> False -- Is this a T_Annotation that ignores a specific code? isAnnotationIgnoringCode code t = diff --git a/src/ShellCheck/Checker.hs b/src/ShellCheck/Checker.hs index f639ab3..673a116 100644 --- a/src/ShellCheck/Checker.hs +++ b/src/ShellCheck/Checker.hs @@ -229,6 +229,12 @@ prop_cantSourceDynamic = prop_cantSourceDynamic2 = [1090] == checkWithIncludes [("lib", "")] "source ~/foo" +prop_canStripPrefixAndSource = + null $ checkWithIncludes [("./lib", "")] "source \"$MYDIR/lib\"" + +prop_canStripPrefixAndSource2 = + null $ checkWithIncludes [("./utils.sh", "")] "source \"$(dirname \"${BASH_SOURCE[0]}\")/utils.sh\"" + prop_canSourceDynamicWhenRedirected = null $ checkWithIncludes [("lib", "")] "#shellcheck source=lib\n. \"$1\"" diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 359fabd..bb27ea3 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -2122,7 +2122,7 @@ readSource t@(T_Redirecting _ _ (T_SimpleCommand cmdId _ (cmd:file':rest'))) = d let file = getFile file' rest' override <- getSourceOverride let literalFile = do - name <- override `mplus` getLiteralString file + name <- override `mplus` getLiteralString file `mplus` stripDynamicPrefix file -- Hack to avoid 'source ~/foo' trying to read from literal tilde guard . not $ "~/" `isPrefixOf` name return name @@ -2182,6 +2182,16 @@ readSource t@(T_Redirecting _ _ (T_SimpleCommand cmdId _ (cmd:file':rest'))) = d SourcePath x -> Just x _ -> Nothing + -- If the word has a single expansion as the directory, try stripping it + -- This affects `$foo/bar` but not `${foo}-dir/bar` or `/foo/$file` + stripDynamicPrefix word = + case getWordParts word of + exp : rest | isStringExpansion exp -> do + str <- getLiteralString (T_NormalWord (Id 0) rest) + guard $ "/" `isPrefixOf` str + return $ "." ++ str + _ -> Nothing + subRead name script = withContext (ContextSource name) $ inSeparateContext $ From baab5b53e09e82f1a4ead133f8ba904458e052ea Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 1 Jul 2020 21:21:38 -0700 Subject: [PATCH 369/763] Use TravisCI workspaces --- .prepare_deploy | 6 +++++- .travis.yml | 38 +++++++++++++++++++++++++++++++++++--- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/.prepare_deploy b/.prepare_deploy index 5e4ffaf..226c16d 100755 --- a/.prepare_deploy +++ b/.prepare_deploy @@ -1,7 +1,7 @@ #!/bin/bash # This script packages up Travis compiled binaries set -ex -shopt -s nullglob +shopt -s nullglob extglob cd deploy cp ../LICENSE LICENSE.txt @@ -59,7 +59,11 @@ do rm "shellcheck" done +rm !(*.xz|*.zip) + for file in ./* do sha512sum "$file" > "$file.sha512sum" done + +ls -l diff --git a/.travis.yml b/.travis.yml index 0ec9718..6e217e8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,14 +7,47 @@ services: jobs: include: - stage: Build - # This must weirdly not have a dash, otherwise an empty job is created env: BUILD=linux - - env: BUILD=windows + workspaces: + create: + name: ws-linux + paths: deploy +# - env: BUILD=windows +# workspaces: +# create: +# name: ws-windows +# paths: deploy - env: BUILD=armv6hf + workspaces: + create: + name: ws-armv6hf + paths: deploy - env: BUILD=aarch64 + workspaces: + create: + name: ws-aarch64 + paths: deploy - env: BUILD=osx os: osx + workspaces: + create: + name: ws-osx + paths: deploy + + - stage: Upload Artifacts to GitHub + workspaces: + use: + - ws-osx + - ws-linux + - ws-armv6hf + - ws-aarch64 + script: + - ls -la ${CASHER_DIR}/ || true + # Kludge broken TravisCI workspaces + - tar -xvf ${CASHER_DIR}/ws-osx-fetch.tgz --strip-components=5 + - ls -la deploy + - ./.github_deploy - stage: Deploy docker image # Deploy only for pushes to master branch, not other branches, not PRs. @@ -39,7 +72,6 @@ script: - ./striptests - set -ex; build_"$BUILD"; set +x; - ./.prepare_deploy - - ./.github_deploy # This is in global context and runs for every stage that doesn't override it. after_failure: | From 9793d94206eb1c8b28a00c2be6a47f6fa0ec7f49 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 5 Jul 2020 20:30:18 -0700 Subject: [PATCH 370/763] Remove trailing whitespace --- .compile_binaries | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.compile_binaries b/.compile_binaries index f657d76..95939ae 100755 --- a/.compile_binaries +++ b/.compile_binaries @@ -16,8 +16,8 @@ build_linux() { ls -l shellcheck ./shellcheck myscript for tag in $TAGS - do - cp "shellcheck" "deploy/shellcheck-$tag.linux-x86_64"; + do + cp "shellcheck" "deploy/shellcheck-$tag.linux-x86_64"; done } @@ -35,8 +35,8 @@ build_armv6hf() { # Linux armv6hf static executable docker run -v "$PWD:/mnt" koalaman/armv6hf-builder -c 'compile-shellcheck' for tag in $TAGS - do - cp "shellcheck" "deploy/shellcheck-$tag.linux-armv6hf"; + do + cp "shellcheck" "deploy/shellcheck-$tag.linux-armv6hf"; done } @@ -44,8 +44,8 @@ build_windows() { # Windows .exe docker run -v "$PWD:/appdata" koalaman/winghc cuib for tag in $TAGS - do - cp "dist/build/ShellCheck/shellcheck.exe" "deploy/shellcheck-$tag.exe"; + do + cp "dist/build/ShellCheck/shellcheck.exe" "deploy/shellcheck-$tag.exe"; done } @@ -66,7 +66,7 @@ build_osx() { [[ -e "$path" ]] for tag in $TAGS - do + do cp "$path" "deploy/shellcheck-$tag.darwin-x86_64"; done } From 7a9dbc042b5b41823fa09f80caedc5bc41deff03 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 5 Jul 2020 20:30:25 -0700 Subject: [PATCH 371/763] Re-enable Windows job --- .travis.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6e217e8..d7d202b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,11 +13,11 @@ jobs: create: name: ws-linux paths: deploy -# - env: BUILD=windows -# workspaces: -# create: -# name: ws-windows -# paths: deploy + - env: BUILD=windows + workspaces: + create: + name: ws-windows + paths: deploy - env: BUILD=armv6hf workspaces: create: @@ -42,6 +42,7 @@ jobs: - ws-linux - ws-armv6hf - ws-aarch64 + - ws-windows script: - ls -la ${CASHER_DIR}/ || true # Kludge broken TravisCI workspaces From 5b86777f9d0ee112b9555002ffe2d834510a25db Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 22 Jul 2020 17:32:00 -0700 Subject: [PATCH 372/763] Warn about non-POSIX case modification expansions (fixes #1977) --- src/ShellCheck/Checks/ShellSupport.hs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 6dd2d39..1f87868 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -178,6 +178,8 @@ prop_checkBashisms93 = verify checkBashisms "#!/bin/sh\necho $(( 10#$(date +%m) prop_checkBashisms94 = verify checkBashisms "#!/bin/sh\n[ -v var ]" prop_checkBashisms95 = verify checkBashisms "#!/bin/sh\necho $_" prop_checkBashisms96 = verifyNot checkBashisms "#!/bin/dash\necho $_" +prop_checkBashisms97 = verify checkBashisms "#!/bin/sh\necho ${var,}" +prop_checkBashisms98 = verify checkBashisms "#!/bin/sh\necho ${var^^}" checkBashisms = ForShell [Sh, Dash] $ \t -> do params <- ask kludge params t @@ -407,6 +409,7 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do (re $ "^![" ++ varChars ++ "]+[*@]$", "name matching prefixes are"), (re $ "^[" ++ varChars ++ "*@]+:[^-=?+]", "string indexing is"), (re $ "^([*@][%#]|#[@*])", "string operations on $@/$* are"), + (re $ "^[" ++ varChars ++ "*@]+(\\[.*\\])?[,^]", "case modification is"), (re $ "^[" ++ varChars ++ "*@]+(\\[.*\\])?/", "string replacement is") ] bashVars = [ From 5d753212fb8385ff80d10f8f8ad96a90e74cfd54 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 25 Jul 2020 13:45:05 -0700 Subject: [PATCH 373/763] Improve handling of command prefixes like exec/command (fixes #2008) --- src/ShellCheck/ASTLib.hs | 73 ++++++++++++++++++++++----- src/ShellCheck/Analytics.hs | 32 +++++++----- src/ShellCheck/AnalyzerLib.hs | 27 ---------- src/ShellCheck/Checks/ShellSupport.hs | 2 +- 4 files changed, 81 insertions(+), 53 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 29ce27f..09c7537 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -28,6 +28,7 @@ import Data.Functor import Data.Functor.Identity import Data.List import Data.Maybe +import qualified Data.Map as Map -- Is this a type of loop? isLoop t = case t of @@ -134,6 +135,33 @@ isUnquotedFlag token = fromMaybe False $ do str <- getLeadingUnquotedString token return $ "-" `isPrefixOf` str +-- getGnuOpts "erd:u:" will parse a SimpleCommand like +-- read -re -d : -u 3 bar +-- into +-- Just [("r", -re), ("e", -re), ("d", :), ("u", 3), ("", bar)] +-- where flags with arguments map to arguments, while others map to themselves. +-- Any unrecognized flag will result in Nothing. +getGnuOpts str t = getOpts str $ getAllFlags t +getBsdOpts str t = getOpts str $ getLeadingFlags t +getOpts :: String -> [(Token, String)] -> Maybe [(String, Token)] +getOpts string flags = process flags + where + flagList (c:':':rest) = ([c], True) : flagList rest + flagList (c:rest) = ([c], False) : flagList rest + flagList [] = [] + flagMap = Map.fromList $ ("", False) : flagList string + + process [] = return [] + process ((token1, flag):rest1) = do + takesArg <- Map.lookup flag flagMap + (token, rest) <- if takesArg + then case rest1 of + (token2, ""):rest2 -> return (token2, rest2) + _ -> fail "takesArg without valid arg" + else return (token1, rest1) + more <- process rest + return $ (flag, token) : more + -- Is this an expansion of multiple items of an array? isArrayExpansion (T_DollarBraced _ _ l) = let string = concat $ oversimplify l in @@ -297,7 +325,7 @@ getCommand t = -- Maybe get the command name string of a token representing a command getCommandName :: Token -> Maybe String -getCommandName = fst . getCommandNameAndToken +getCommandName = fst . getCommandNameAndToken False -- Maybe get the name+arguments of a command. getCommandArgv t = do @@ -307,18 +335,37 @@ getCommandArgv t = do -- Get the command name token from a command, i.e. -- the token representing 'ls' in 'ls -la 2> foo'. -- If it can't be determined, return the original token. -getCommandTokenOrThis = snd . getCommandNameAndToken +getCommandTokenOrThis = snd . getCommandNameAndToken False -getCommandNameAndToken :: Token -> (Maybe String, Token) -getCommandNameAndToken t = fromMaybe (Nothing, t) $ do - (T_SimpleCommand _ _ (w:rest)) <- getCommand t +-- Given a command, get the string and token that represents the command name. +-- If direct, return the actual command (e.g. exec in 'exec ls') +-- If not, return the logical command (e.g. 'ls' in 'exec ls') + +getCommandNameAndToken :: Bool -> Token -> (Maybe String, Token) +getCommandNameAndToken direct t = fromMaybe (Nothing, t) $ do + cmd@(T_SimpleCommand _ _ (w:rest)) <- getCommand t s <- getLiteralString w - return $ case rest of - (applet:_) | "busybox" `isSuffixOf` s || "builtin" == s -> - (getLiteralString applet, applet) - _ -> - (Just s, w) - + return $ fromMaybe (Just s, w) $ do + guard $ not direct + actual <- getEffectiveCommandToken s cmd rest + return (getLiteralString actual, actual) + where + getEffectiveCommandToken str cmd args = + let + firstArg = do + arg <- listToMaybe args + guard . not $ isFlag arg + return arg + in + case str of + "busybox" -> firstArg + "builtin" -> firstArg + "command" -> firstArg + "exec" -> do + opts <- getBsdOpts "cla:" cmd + (_, t) <- listToMaybe $ filter (null . fst) opts + return t + _ -> fail "" -- If a command substitution is a single command, get its name. -- $(date +%s) = Just "date" @@ -335,8 +382,8 @@ getCommandNameFromExpansion t = -- Get the basename of a token representing a command getCommandBasename = fmap basename . getCommandName - where - basename = reverse . takeWhile (/= '/') . reverse + +basename = reverse . takeWhile (/= '/') . reverse isAssignment t = case t of diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 977d415..4071764 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -942,8 +942,10 @@ prop_checkSingleQuotedVariables18= verifyNot checkSingleQuotedVariables "echo '` prop_checkSingleQuotedVariables19= verifyNot checkSingleQuotedVariables "echo '```'" prop_checkSingleQuotedVariables20= verifyNot checkSingleQuotedVariables "mumps -run %XCMD 'W $O(^GLOBAL(5))'" prop_checkSingleQuotedVariables21= verifyNot checkSingleQuotedVariables "mumps -run LOOP%XCMD --xec 'W $O(^GLOBAL(6))'" - - +prop_checkSingleQuotedVariables22= verifyNot checkSingleQuotedVariables "jq '$__loc__'" +prop_checkSingleQuotedVariables23= verifyNot checkSingleQuotedVariables "command jq '$__loc__'" +prop_checkSingleQuotedVariables24= verifyNot checkSingleQuotedVariables "exec jq '$__loc__'" +prop_checkSingleQuotedVariables25= verifyNot checkSingleQuotedVariables "exec -c -a foo jq '$__loc__'" checkSingleQuotedVariables params t@(T_SingleQuoted id s) = @@ -1677,13 +1679,10 @@ checkSpuriousExec _ = doLists doList tail True doList' _ _ = return () - commentIfExec (T_Pipeline id _ list) = - mapM_ commentIfExec $ take 1 list - commentIfExec (T_Redirecting _ _ f@( - T_SimpleCommand id _ (cmd:arg:_))) - | f `isUnqualifiedCommand` "exec" = - warn id 2093 - "Remove \"exec \" if script should continue after this command." + commentIfExec (T_Pipeline id _ [c]) = commentIfExec c + commentIfExec (T_Redirecting _ _ (T_SimpleCommand id _ (cmd:additionalArg:_))) | + getLiteralString cmd == Just "exec" = + warn id 2093 "Remove \"exec \" if script should continue after this command." commentIfExec _ = return () @@ -2056,18 +2055,27 @@ prop_checkFunctionsUsedExternally6 = verifyNotTree checkFunctionsUsedExternally "foo() { :; }; ssh host echo foo" prop_checkFunctionsUsedExternally7 = verifyNotTree checkFunctionsUsedExternally "install() { :; }; sudo apt-get install foo" +prop_checkFunctionsUsedExternally8 = + verifyTree checkFunctionsUsedExternally "foo() { :; }; command sudo foo" +prop_checkFunctionsUsedExternally9 = + verifyTree checkFunctionsUsedExternally "foo() { :; }; exec -c sudo foo" checkFunctionsUsedExternally params t = runNodeAnalysis checkCommand params t where - checkCommand _ t@(T_SimpleCommand _ _ (cmd:args)) = - case getCommandBasename t of - Just name -> do + checkCommand _ t@(T_SimpleCommand _ _ argv) = + case getCommandNameAndToken False t of + (Just str, t) -> do + let name = basename str + let args = skipOver t argv let argStrings = map (\x -> (fromMaybe "" $ getLiteralString x, x)) args let candidates = getPotentialCommands name argStrings mapM_ (checkArg name) candidates _ -> return () checkCommand _ _ = return () + skipOver t list = drop 1 $ dropWhile (\c -> getId c /= id) $ list + where id = getId t + -- Try to pick out the argument[s] that may be commands getPotentialCommands name argAndString = case name of diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 69ebfe6..ab1c415 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -901,33 +901,6 @@ isQuotedAlternativeReference t = where re = mkRegex "(^|\\]):?\\+" --- getGnuOpts "erd:u:" will parse a SimpleCommand like --- read -re -d : -u 3 bar --- into --- Just [("r", -re), ("e", -re), ("d", :), ("u", 3), ("", bar)] --- where flags with arguments map to arguments, while others map to themselves. --- Any unrecognized flag will result in Nothing. -getGnuOpts str t = getOpts str $ getAllFlags t -getBsdOpts str t = getOpts str $ getLeadingFlags t -getOpts :: String -> [(Token, String)] -> Maybe [(String, Token)] -getOpts string flags = process flags - where - flagList (c:':':rest) = ([c], True) : flagList rest - flagList (c:rest) = ([c], False) : flagList rest - flagList [] = [] - flagMap = Map.fromList $ ("", False) : flagList string - - process [] = return [] - process ((token1, flag):rest1) = do - takesArg <- Map.lookup flag flagMap - (token, rest) <- if takesArg - then case rest1 of - (token2, ""):rest2 -> return (token2, rest2) - _ -> fail "takesArg without valid arg" - else return (token1, rest1) - more <- process rest - return $ (flag, token) : more - supportsArrays Bash = True supportsArrays Ksh = True supportsArrays _ = False diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 1f87868..2482207 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -280,7 +280,7 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do flagRegex = mkRegex "^-[eEsn]+$" bashism t@(T_SimpleCommand _ _ (cmd:arg:_)) - | t `isCommand` "exec" && "-" `isPrefixOf` concat (oversimplify arg) = + | getLiteralString cmd == Just "exec" && "-" `isPrefixOf` concat (oversimplify arg) = warnMsg (getId arg) "exec flags are" bashism t@(T_SimpleCommand id _ _) | t `isCommand` "let" = warnMsg id "'let' is" From 14e680609271ca77b9f1d49ca7fa44787b98b4b2 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 25 Jul 2020 17:36:22 -0700 Subject: [PATCH 374/763] Handle literal linefeeds in printf format strings (fixes #2007) --- src/ShellCheck/Checks/Commands.hs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 63e58a1..b4e8b3e 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -560,6 +560,8 @@ prop_checkPrintfVar18= verifyNot checkPrintfVar "printf '%-*s\\n' 1 2" prop_checkPrintfVar19= verifyNot checkPrintfVar "printf '%(%s)T'" prop_checkPrintfVar20= verifyNot checkPrintfVar "printf '%d %(%s)T' 42" prop_checkPrintfVar21= verify checkPrintfVar "printf '%d %(%s)T'" +prop_checkPrintfVar22= verify checkPrintfVar "printf '%s\n%s' foo" + checkPrintfVar = CommandCheck (Exactly "printf") (f . arguments) where f (doubledash:rest) | getLiteralString doubledash == Just "--" = f rest f (dashv:var:rest) | getLiteralString dashv == Just "-v" = f rest @@ -602,6 +604,8 @@ prop_checkGetPrintfFormats2 = getPrintfFormats "%0*s" == "*s" prop_checkGetPrintfFormats3 = getPrintfFormats "%(%s)T" == "T" prop_checkGetPrintfFormats4 = getPrintfFormats "%d%%%(%s)T" == "dT" prop_checkGetPrintfFormats5 = getPrintfFormats "%bPassed: %d, %bFailed: %d%b, Skipped: %d, %bErrored: %d%b\\n" == "bdbdbdbdb" +prop_checkGetPrintfFormats6 = getPrintfFormats "%s%s" == "ss" +prop_checkGetPrintfFormats7 = getPrintfFormats "%s\n%s" == "ss" getPrintfFormats = getFormats where -- Get the arguments in the string as a string of type characters, @@ -620,17 +624,17 @@ getPrintfFormats = getFormats regexBasedGetFormats rest = case matchRegex re rest of - Just [width, precision, typ, rest] -> + Just [width, precision, typ, rest, _] -> (if width == "*" then "*" else "") ++ (if precision == "*" then "*" else "") ++ typ ++ getFormats rest Nothing -> take 1 rest ++ getFormats rest where -- constructed based on specifications in "man printf" - re = mkRegex "#?-?\\+? ?0?(\\*|\\d*)\\.?(\\d*|\\*)([diouxXfFeEgGaAcsbq])(.*)" - -- \____ _____/\___ ____/ \____ ____/\_________ _________/ \ / - -- V V V V V - -- flags field width precision format character rest + re = mkRegex "#?-?\\+? ?0?(\\*|\\d*)\\.?(\\d*|\\*)([diouxXfFeEgGaAcsbq])((\n|.)*)" + -- \____ _____/\___ ____/ \____ ____/\_________ _________/ \______ / + -- V V V V V + -- flags field width precision format character rest -- field width and precision can be specified with a '*' instead of a digit, -- in which case printf will accept one more argument for each '*' used From 34885142e795fccce0a3f4c19329ca2badf09c12 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 27 Jul 2020 18:34:42 -0700 Subject: [PATCH 375/763] Handle tilde expansion in pattern matching (fixes #1769) --- src/ShellCheck/ASTLib.hs | 39 +++++++++++++++++++++---------------- src/ShellCheck/Analytics.hs | 3 +++ 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 09c7537..29feb1e 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -463,30 +463,35 @@ data PseudoGlob = PGAny | PGMany | PGChar Char -- Turn a word into a PG pattern, replacing all unknown/runtime values with -- PGMany. wordToPseudoGlob :: Token -> [PseudoGlob] -wordToPseudoGlob word = - simplifyPseudoGlob . concatMap f $ getWordParts word - where - f x = case x of - T_Literal _ s -> map PGChar s - T_SingleQuoted _ s -> map PGChar s - - T_Glob _ "?" -> [PGAny] - T_Glob _ ('[':_) -> [PGAny] - - _ -> [PGMany] +wordToPseudoGlob = fromMaybe [PGMany] . wordToPseudoGlob' False -- Turn a word into a PG pattern, but only if we can preserve -- exact semantics. wordToExactPseudoGlob :: Token -> Maybe [PseudoGlob] -wordToExactPseudoGlob word = - simplifyPseudoGlob . concat <$> mapM f (getWordParts word) +wordToExactPseudoGlob = wordToPseudoGlob' True + +wordToPseudoGlob' :: Bool -> Token -> Maybe [PseudoGlob] +wordToPseudoGlob' exact word = + simplifyPseudoGlob <$> toGlob word where + toGlob :: Token -> Maybe [PseudoGlob] + toGlob word = + case word of + T_NormalWord _ (T_Literal _ ('~':str):rest) -> do + guard $ not exact + let this = (PGMany : (map PGChar $ dropWhile (/= '/') str)) + tail <- concat <$> (mapM f $ concatMap getWordParts rest) + return $ this ++ tail + _ -> concat <$> (mapM f $ getWordParts word) + f x = case x of - T_Literal _ s -> return $ map PGChar s + T_Literal _ s -> return $ map PGChar s T_SingleQuoted _ s -> return $ map PGChar s - T_Glob _ "?" -> return [PGAny] - T_Glob _ "*" -> return [PGMany] - _ -> fail "Unknown token type" + T_Glob _ "?" -> return [PGAny] + T_Glob _ "*" -> return [PGMany] + T_Glob _ ('[':_) | not exact -> return [PGAny] + _ -> if exact then fail "" else return [PGMany] + -- Reorder a PseudoGlob for more efficient matching, e.g. -- f?*?**g -> f??*g diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 4071764..bba3caf 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1211,6 +1211,9 @@ prop_checkConstantIfs6 = verifyNot checkConstantIfs "[[ a -ot b ]]" prop_checkConstantIfs7 = verifyNot checkConstantIfs "[ a -nt b ]" prop_checkConstantIfs8 = verifyNot checkConstantIfs "[[ ~foo == '~foo' ]]" prop_checkConstantIfs9 = verify checkConstantIfs "[[ *.png == [a-z] ]]" +prop_checkConstantIfs10 = verifyNot checkConstantIfs "[[ ~me == ~+ ]]" +prop_checkConstantIfs11 = verifyNot checkConstantIfs "[[ ~ == ~+ ]]" +prop_checkConstantIfs12 = verify checkConstantIfs "[[ '~' == x ]]" checkConstantIfs _ (TC_Binary id typ op lhs rhs) | not isDynamic = if isConstant lhs && isConstant rhs then warn id 2050 "This expression is constant. Did you forget the $ on a variable?" From cc81bdee31fdf6e9ff90be19f8619307db401358 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 27 Jul 2020 18:44:07 -0700 Subject: [PATCH 376/763] Improve SC1033/SC1034 message --- src/ShellCheck/Parser.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index bb27ea3..e5b10f4 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -945,8 +945,8 @@ readCondition = called "test expression" $ do cpos <- getPosition close <- try (string "]]") <|> string "]" <|> fail "Expected test to end here (don't wrap commands in []/[[]])" id <- endSpan start - when (open == "[[" && close /= "]]") $ parseProblemAt cpos ErrorC 1033 "Did you mean ]] ?" - when (open == "[" && close /= "]" ) $ parseProblemAt opos ErrorC 1034 "Did you mean [[ ?" + when (open == "[[" && close /= "]]") $ parseProblemAt cpos ErrorC 1033 "Test expression was opened with double [[ but closed with single ]. Make sure they match." + when (open == "[" && close /= "]" ) $ parseProblemAt opos ErrorC 1034 "Test expression was opened with single [ but closed with double ]]. Make sure they match." optional $ lookAhead $ do pos <- getPosition notFollowedBy2 readCmdWord <|> From 1ac2c31728dd3109aea3c120d8fa021d358319af Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 27 Jul 2020 21:28:48 -0700 Subject: [PATCH 377/763] Warn when shell functions blatantly recurse (fixes #1994) --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 37 +++++++++++++++++++++++++++++++++++ src/ShellCheck/AnalyzerLib.hs | 2 ++ 3 files changed, 40 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fde8b0f..a9ea5f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ - SC2259/SC2260: Warn when redirections override pipes - SC2261: Warn about multiple competing redirections - SC2262/SC2263: Warn about aliases declared and used in the same parsing unit +- SC2264: Warn about wrapper functions that blatantly recurse ### Fixed - SC1072/SC1073 now respond to disable annotations, though ignoring parse errors diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index bba3caf..f8904be 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -191,6 +191,7 @@ nodeChecks = [ ,checkUselessBang ,checkTranslatedStringVariable ,checkModifiedArithmeticInRedirection + ,checkBlatantRecursion ] optionalChecks = map fst optionalTreeChecks @@ -3824,5 +3825,41 @@ groupByLink f list = else (reverse $ current:span) : g next [] rest g current span [] = [reverse (current:span)] + +prop_checkBlatantRecursion1 = verify checkBlatantRecursion ":(){ :|:& };:" +prop_checkBlatantRecursion2 = verify checkBlatantRecursion "f() { f; }" +prop_checkBlatantRecursion3 = verifyNot checkBlatantRecursion "f() { command f; }" +prop_checkBlatantRecursion4 = verify checkBlatantRecursion "cd() { cd \"$lol/$1\" || exit; }" +prop_checkBlatantRecursion5 = verifyNot checkBlatantRecursion "cd() { [ -z \"$1\" ] || cd \"$1\"; }" +prop_checkBlatantRecursion6 = verifyNot checkBlatantRecursion "cd() { something; cd $1; }" +prop_checkBlatantRecursion7 = verifyNot checkBlatantRecursion "cd() { builtin cd $1; }" +checkBlatantRecursion :: Parameters -> Token -> Writer [TokenComment] () +checkBlatantRecursion params t = + case t of + T_Function _ _ _ name body -> + case getCommandSequences body of + [first : _] -> checkList name first + _ -> return () + _ -> return () + where + checkList :: String -> Token -> Writer [TokenComment] () + checkList name t = + case t of + T_Backgrounded _ t -> checkList name t + T_AndIf _ lhs _ -> checkList name lhs + T_OrIf _ lhs _ -> checkList name lhs + T_Pipeline _ _ cmds -> mapM_ (checkCommand name) cmds + _ -> return () + + checkCommand :: String -> Token -> Writer [TokenComment] () + checkCommand name cmd = sequence_ $ do + let (invokedM, t) = getCommandNameAndToken True cmd + invoked <- invokedM + guard $ name == invoked + return $ + errWithFix (getId t) 2264 + ("This function unconditionally re-invokes itself. Missing 'command'?") + (fixWith [replaceStart (getId t) params 0 $ "command "]) + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index ab1c415..1ff50b2 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -163,6 +163,8 @@ err id code str = addComment $ makeComment ErrorC id code str info id code str = addComment $ makeComment InfoC id code str style id code str = addComment $ makeComment StyleC id code str +errWithFix :: MonadWriter [TokenComment] m => Id -> Code -> String -> Fix -> m () +errWithFix = addCommentWithFix ErrorC warnWithFix :: MonadWriter [TokenComment] m => Id -> Code -> String -> Fix -> m () warnWithFix = addCommentWithFix WarningC styleWithFix :: MonadWriter [TokenComment] m => Id -> Code -> String -> Fix -> m () From beee9b22cadead5066b58c0bb67047d142f7c6f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9E=97=E5=8D=9A=E4=BB=81=28Buo-ren=2C=20Lin=29?= Date: Sun, 2 Aug 2020 16:55:45 +0800 Subject: [PATCH 378/763] Fix snap distribution unable to process scripts in Unicode(Chinese) (fixes #1643) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The snap runtime only supports C.UTF-8 locale, as other locales don't seem to be used now just hardcode it. Fixes #1643. Signed-off-by: 林博仁(Buo-ren, Lin) --- snap/snapcraft.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 8365b13..2dc4831 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -31,6 +31,8 @@ apps: shellcheck: command: usr/bin/shellcheck plugs: [home, removable-media] + environment: + LANG: C.UTF-8 parts: shellcheck: From c5b6d6f027c700a1ab075c5f84da5aa7a6ca0df8 Mon Sep 17 00:00:00 2001 From: Brennan Vincent Date: Wed, 5 Aug 2020 10:50:14 -0400 Subject: [PATCH 379/763] Update README.md Minor typographical fixes ("macOS" everywhere, capitalize Homebrew) --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index bfb3a99..6f38fa5 100644 --- a/README.md +++ b/README.md @@ -167,7 +167,7 @@ On FreeBSD: pkg install hs-ShellCheck -On OS X with homebrew: +On macOS (OS X) with Homebrew: brew install shellcheck @@ -220,7 +220,7 @@ Alternatively, you can download pre-compiled binaries for the latest release her * [Linux, x86_64](https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.x86_64.tar.xz) (statically linked) * [Linux, armv6hf](https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.armv6hf.tar.xz), i.e. Raspberry Pi (statically linked) * [Linux, aarch64](https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.aarch64.tar.xz) aka ARM64 (statically linked) -* [MacOS, x86_64](https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.darwin.x86_64.tar.xz) +* [macOS, x86_64](https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.darwin.x86_64.tar.xz) * [Windows, x86](https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.zip) or see the [GitHub Releases](https://github.com/koalaman/shellcheck/releases) for other releases @@ -264,7 +264,7 @@ This section describes how to build ShellCheck from a source directory. ShellChe ShellCheck is built and packaged using Cabal. Install the package `cabal-install` from your system's package manager (with e.g. `apt-get`, `brew`, `emerge`, `yum`, or `zypper`). -On MacOS (OS X), you can do a fast install of Cabal using brew, which takes a couple of minutes instead of more than 30 minutes if you try to compile it from source. +On macOS (OS X), you can do a fast install of Cabal using brew, which takes a couple of minutes instead of more than 30 minutes if you try to compile it from source. $ brew install cabal-install From 3e50a2fce8323f89b63243d2b9042d756d327920 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Fri, 7 Aug 2020 14:59:34 -0700 Subject: [PATCH 380/763] Suppress SC2216 for du --files0-from or --exclude-from (fixes #1286) --- src/ShellCheck/Analytics.hs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index f8904be..7906e7a 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -3305,6 +3305,8 @@ prop_checkPipeToNowhere15 = verifyNot checkPipeToNowhere "ls > foo 2> bar |& gre prop_checkPipeToNowhere16 = verifyNot checkPipeToNowhere "echo World | cat << EOF\nhello $(cat)\nEOF\n" prop_checkPipeToNowhere17 = verify checkPipeToNowhere "echo World | cat << 'EOF'\nhello $(cat)\nEOF\n" prop_checkPipeToNowhere18 = verifyNot checkPipeToNowhere "ls 1>&3 3>&1 3>&- | wc -l" +prop_checkPipeToNowhere19 = verifyNot checkPipeToNowhere "find . -print0 | du --files0-from=/dev/stdin" +prop_checkPipeToNowhere20 = verifyNot checkPipeToNowhere "find . | du --exclude-from=/dev/fd/0" data PipeType = StdoutPipe | StdoutStderrPipe | NoPipe deriving (Eq) checkPipeToNowhere :: Parameters -> Token -> WriterT [TokenComment] Identity () @@ -3324,6 +3326,7 @@ checkPipeToNowhere params t = name <- getCommandBasename cmd guard $ name `elem` nonReadingCommands guard $ not hasConsumers && input /= NoPipe + guard . not $ commandSpecificException name cmd -- Confusing echo for cat is so common that it's worth a special case let suggestion = @@ -3366,6 +3369,11 @@ checkPipeToNowhere params t = outputWarning mapM_ warnAboutDupes $ Map.assocs fdMap + commandSpecificException name cmd = + case name of + "du" -> any (`elem` ["exclude-from", "files0-from"]) $ lt $ map snd $ getAllFlags cmd + _ -> False + warnAboutDupes (n, list@(_:_:_)) = forM_ list $ \c -> err (getOpId c) 2261 $ "Multiple redirections compete for " ++ str n ++ ". Use cat, tee, or pass filenames instead." From e779aedac3355bba0c6d071c741845d62d7aa4d8 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Fri, 7 Aug 2020 16:00:58 -0700 Subject: [PATCH 381/763] Modernize getting mapfile array name --- src/ShellCheck/Analytics.hs | 4 +++- src/ShellCheck/AnalyzerLib.hs | 35 ++++++++++++++++++++++++----------- 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 9ad53ae..f5c1a18 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2228,7 +2228,9 @@ prop_checkUnassignedReferences37= verifyNotTree checkUnassignedReferences "var=h prop_checkUnassignedReferences38= verifyTree (checkUnassignedReferences' True) "echo $VAR" prop_checkUnassignedReferences39= verifyNotTree checkUnassignedReferences "builtin export var=4; echo $var" prop_checkUnassignedReferences40= verifyNotTree checkUnassignedReferences ": ${foo=bar}" -prop_checkUnassignedReferences41= verifyNotTree checkUnassignedReferences "mapfile -t files <(cat); echo \"${files[@]}\"" +prop_checkUnassignedReferences41= verifyNotTree checkUnassignedReferences "mapfile -t files 123; echo \"${files[@]}\"" +prop_checkUnassignedReferences42= verifyNotTree checkUnassignedReferences "mapfile files -t; echo \"${files[@]}\"" +prop_checkUnassignedReferences43= verifyNotTree checkUnassignedReferences "mapfile --future files; echo \"${files[@]}\"" checkUnassignedReferences = checkUnassignedReferences' False checkUnassignedReferences' includeGlobals params t = warnings diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index ba2c901..596340b 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -658,17 +658,30 @@ getModifiedVariableCommand base@(T_SimpleCommand id cmdPrefix (T_NormalWord _ (T f [] = fail "not found" -- mapfile has some curious syntax allowing flags plus 0..n variable names - -- where only the first non-option one is used if any. Here we cheat and - -- just get the last one, if it's a variable name, and omitting process - -- substitions. - getMapfileArray base arguments = do - lastArg <- listToMaybe (filter notProcSub $ reverse arguments) - name <- getLiteralString lastArg - guard $ isVariableName name - return (base, lastArg, name, DataArray SourceExternal) - - notProcSub (T_NormalWord _ (T_ProcSub{} :_)) = False - notProcSub _ = True + -- where only the first non-option one is used if any. + getMapfileArray base rest = parseArgs `mplus` fallback + where + parseArgs :: Maybe (Token, Token, String, DataType) + parseArgs = do + args <- getGnuOpts "d:n:O:s:u:C:c:t" base + let names = map snd $ filter (\(x,y) -> null x) args + if null names + then + return (base, base, "MAPFILE", DataArray SourceExternal) + else do + first <- listToMaybe names + name <- getLiteralString first + guard $ isVariableName name + return (base, first, name, DataArray SourceExternal) + -- If arg parsing fails (due to bad or new flags), get the last variable name + fallback :: Maybe (Token, Token, String, DataType) + fallback = do + (name, token) <- listToMaybe . mapMaybe f $ reverse rest + return (base, token, name, DataArray SourceExternal) + f arg = do + name <- getLiteralString arg + guard $ isVariableName name + return (name, arg) -- get all the array variables used in read, e.g. read -a arr getReadArrayVariables args = From 50067ddf94c04b2835a44a0ec16e3772e0a01835 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 8 Aug 2020 12:32:20 -0700 Subject: [PATCH 382/763] Consider variables in -z/-n tests to be checked --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 4 ---- src/ShellCheck/AnalyzerLib.hs | 21 ++++++++++++--------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a9ea5f4..1b3a2f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ ### Changed - SC1090: A leading `$x/` or `$(x)/` is now treated as `./` when locating files +- SC2154: Variables appearing in -z/-n tests are no longer considered unassigned ## v0.7.1 - 2020-04-04 diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 40f6375..9b92698 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2231,7 +2231,6 @@ prop_checkUnassignedReferences40= verifyNotTree checkUnassignedReferences ": ${f prop_checkUnassignedReferences41= verifyNotTree checkUnassignedReferences "mapfile -t files 123; echo \"${files[@]}\"" prop_checkUnassignedReferences42= verifyNotTree checkUnassignedReferences "mapfile files -t; echo \"${files[@]}\"" prop_checkUnassignedReferences43= verifyNotTree checkUnassignedReferences "mapfile --future files; echo \"${files[@]}\"" - prop_checkUnassignedReferences_minusNPlain = verifyNotTree checkUnassignedReferences "if [ -n \"$x\" ]; then echo $x; fi" prop_checkUnassignedReferences_minusZPlain = verifyNotTree checkUnassignedReferences "if [ -z \"$x\" ]; then echo \"\"; fi" prop_checkUnassignedReferences_minusNBraced = verifyNotTree checkUnassignedReferences "if [ -n \"${x}\" ]; then echo $x; fi" @@ -2239,9 +2238,6 @@ prop_checkUnassignedReferences_minusZBraced = verifyNotTree checkUnassignedRefe prop_checkUnassignedReferences_minusNDefault = verifyNotTree checkUnassignedReferences "if [ -n \"${x:-}\" ]; then echo $x; fi" prop_checkUnassignedReferences_minusZDefault = verifyNotTree checkUnassignedReferences "if [ -z \"${x:-}\" ]; then echo \"\"; fi" -prop_checkUnassignedReferences_minusZInsteadOfN = verifyTree checkUnassignedReferences "if [ -z \"$x\" ]; then echo $x; fi" -prop_checkUnassignedReferences_minusZInsteadOfNBraced = verifyTree checkUnassignedReferences "if [ -z \"${x}\" ]; then echo $x; fi" - checkUnassignedReferences = checkUnassignedReferences' False checkUnassignedReferences' includeGlobals params t = warnings where diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index d87362c..c9954d1 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -499,8 +499,9 @@ getModifiedVariables t = guard . not . null $ str return (t, token, str, DataString SourceChecked) - TC_Unary _ _ "-n" (T_NormalWord _ [T_DoubleQuoted _ [db@(T_DollarBraced _ _ l)]]) -> - [(t, t, getBracedReference (concat $ oversimplify l), DataString SourceChecked)] + TC_Unary _ _ "-n" token -> markAsChecked t token + TC_Unary _ _ "-z" token -> markAsChecked t token + TC_Nullary _ _ token -> markAsChecked t token T_DollarBraced _ _ l -> maybeToList $ do let string = concat $ oversimplify l @@ -519,6 +520,14 @@ getModifiedVariables t = T_ForIn id str words _ -> [(t, t, str, DataString $ SourceFrom words)] T_SelectIn id str words _ -> [(t, t, str, DataString $ SourceFrom words)] _ -> [] + where + markAsChecked place token = mapMaybe (f place) $ getWordParts token + f place t = case t of + T_DollarBraced _ _ l -> + let str = getBracedReference $ concat $ oversimplify l in do + guard $ isVariableName str + return (place, t, str, DataString SourceChecked) + _ -> Nothing isClosingFileOp op = case op of @@ -726,9 +735,7 @@ getOffsetReferences mods = fromMaybe [] $ do getReferencedVariables parents t = case t of T_DollarBraced id _ l -> let str = concat $ oversimplify l in - if isMinusZTest t - then [] - else (t, t, getBracedReference str) : + (t, t, getBracedReference str) : map (\x -> (l, l, x)) ( getIndexReferences str ++ getOffsetReferences (getBracedModifier str)) @@ -782,10 +789,6 @@ getReferencedVariables parents t = this: TA_Assignment _ "=" lhs _ :_ -> lhs == t _ -> False - isMinusZTest t = case getPath parents t of - _ : T_DoubleQuoted _ [_] : T_NormalWord _ [_] : TC_Unary _ SingleBracket "-z" _ : _ -> True - _ -> False - dataTypeFrom defaultType v = (case v of T_Array {} -> DataArray; _ -> defaultType) $ SourceFrom [v] From e72fbb2640a9aee77d1e380c6f9364fd2b500bb9 Mon Sep 17 00:00:00 2001 From: Simon Shine Date: Thu, 20 Aug 2020 13:07:32 +0200 Subject: [PATCH 383/763] Fix whitespace in README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e400651..05ce5ad 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ Services and platforms that have ShellCheck pre-installed and ready to use: * [Code Climate](https://codeclimate.com/) * [Code Factor](https://www.codefactor.io/) * [CircleCI](https://circleci.com) via the [ShellCheck Orb](https://circleci.com/orbs/registry/orb/circleci/shellcheck) -* [Github](https://github.com/features/actions)(only Linux) +* [Github](https://github.com/features/actions) (only Linux) Services and platforms with third party plugins: From a62d9f10c29587056cdad30942b984c3e492dfc1 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 23 Aug 2020 15:43:33 -0700 Subject: [PATCH 384/763] Warn when using &/| between test statements --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 46 +++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b3a2f0..3d91611 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - SC2261: Warn about multiple competing redirections - SC2262/SC2263: Warn about aliases declared and used in the same parsing unit - SC2264: Warn about wrapper functions that blatantly recurse +- SC2265/SC2266: Warn when using & or | with test statements ### Fixed - SC1072/SC1073 now respond to disable annotations, though ignoring parse errors diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 9b92698..ffcf9cf 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -192,6 +192,7 @@ nodeChecks = [ ,checkTranslatedStringVariable ,checkModifiedArithmeticInRedirection ,checkBlatantRecursion + ,checkBadTestAndOr ] optionalChecks = map fst optionalTreeChecks @@ -3878,5 +3879,50 @@ checkBlatantRecursion params t = ("This function unconditionally re-invokes itself. Missing 'command'?") (fixWith [replaceStart (getId t) params 0 $ "command "]) + +prop_checkBadTestAndOr1 = verify checkBadTestAndOr "[ x ] & [ y ]" +prop_checkBadTestAndOr2 = verify checkBadTestAndOr "test -e foo & [ y ]" +prop_checkBadTestAndOr3 = verify checkBadTestAndOr "[ x ] | [ y ]" +checkBadTestAndOr params t = + case t of + T_Pipeline _ seps cmds@(_:_:_) -> checkOrs seps cmds + T_Backgrounded id cmd -> checkAnds id cmd + _ -> return () + where + checkOrs seps cmds = + let maybeSeps = map Just seps + commandWithSeps = zip3 (Nothing:maybeSeps) cmds (maybeSeps ++ [Nothing]) + in + mapM_ checkTest commandWithSeps + checkTest (before, cmd, after) = + when (isTest cmd) $ do + checkPipe before + checkPipe after + + checkPipe t = + case t of + Just (T_Pipe id "|") -> + warnWithFix id 2266 "Use || for logical OR. Single | will pipe." $ + fixWith [replaceEnd id params 0 "|"] + _ -> return () + + checkAnds id t = + case t of + T_AndIf _ _ rhs -> checkAnds id rhs + T_OrIf _ _ rhs -> checkAnds id rhs + T_Pipeline _ _ list | not (null list) -> checkAnds id (last list) + cmd -> when (isTest cmd) $ + errWithFix id 2265 "Use && for logical AND. Single & will background and return true." $ + (fixWith [replaceEnd id params 0 "&"]) + + isTest t = + case t of + T_Condition {} -> True + T_SimpleCommand {} -> t `isCommand` "test" + T_Redirecting _ _ t -> isTest t + T_Annotation _ _ t -> isTest t + _ -> False + + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) From 9e59bcca9131359b9f418b209cbfbe41116a3679 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 23 Aug 2020 15:49:20 -0700 Subject: [PATCH 385/763] Upgrade SC2169 (unsupported in dash) from warning to error (fixes #2013) --- src/ShellCheck/Checks/ShellSupport.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 2482207..788323d 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -190,7 +190,7 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do isDash = shellType params == Dash warnMsg id s = if isDash - then warn id 2169 $ "In dash, " ++ s ++ " not supported." + then err id 2169 $ "In dash, " ++ s ++ " not supported." else warn id 2039 $ "In POSIX sh, " ++ s ++ " undefined." bashism (T_ProcSub id _ _) = warnMsg id "process substitution is" From c9be7ab2eb7ccb05ca0a5cb75946469a3cee98e0 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 23 Aug 2020 18:46:13 -0700 Subject: [PATCH 386/763] Parse assignments according to spec (fixes #2022) --- CHANGELOG.md | 1 + src/ShellCheck/Parser.hs | 95 +++++++++++++++++++++++++--------------- 2 files changed, 60 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d91611..5db445b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ is still purely cosmetic and does not allow ShellCheck to continue. ### Changed +- Assignments are now parsed to spec, without leniency for leading $ or spaces - SC1090: A leading `$x/` or `$(x)/` is now treated as `./` when locating files - SC2154: Variables appearing in -z/-n tests are no longer considered unassigned diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index e5b10f4..4806b18 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -212,6 +212,12 @@ endSpan (IncompleteInterval start) = do id <- getNextIdBetween start endPos return id +getSpanPositionsFor m = do + start <- getPosition + m + end <- getPosition + return (start, end) + addToHereDocMap id list = do state <- getState let map = hereDocMap state @@ -2788,17 +2794,13 @@ readLiteralForParser parser = do prop_readAssignmentWord = isOk readAssignmentWord "a=42" prop_readAssignmentWord2 = isOk readAssignmentWord "b=(1 2 3)" -prop_readAssignmentWord3 = isWarning readAssignmentWord "$b = 13" -prop_readAssignmentWord4 = isWarning readAssignmentWord "b = $(lol)" prop_readAssignmentWord5 = isOk readAssignmentWord "b+=lol" -prop_readAssignmentWord6 = isWarning readAssignmentWord "b += (1 2 3)" prop_readAssignmentWord7 = isOk readAssignmentWord "a[3$n'']=42" prop_readAssignmentWord8 = isOk readAssignmentWord "a[4''$(cat foo)]=42" prop_readAssignmentWord9 = isOk readAssignmentWord "IFS= " prop_readAssignmentWord9a= isOk readAssignmentWord "foo=" prop_readAssignmentWord9b= isOk readAssignmentWord "foo= " prop_readAssignmentWord9c= isOk readAssignmentWord "foo= #bar" -prop_readAssignmentWord10= isWarning readAssignmentWord "foo$n=42" prop_readAssignmentWord11= isOk readAssignmentWord "foo=([a]=b [c] [d]= [e f )" prop_readAssignmentWord12= isOk readAssignmentWord "a[b <<= 3 + c]='thing'" prop_readAssignmentWord13= isOk readAssignmentWord "var=( (1 2) (3 4) )" @@ -2806,52 +2808,73 @@ prop_readAssignmentWord14= isOk readAssignmentWord "var=( 1 [2]=(3 4) )" prop_readAssignmentWord15= isOk readAssignmentWord "var=(1 [2]=(3 4))" readAssignmentWord = readAssignmentWordExt True readWellFormedAssignment = readAssignmentWordExt False -readAssignmentWordExt lenient = try $ do - start <- startSpan - pos <- getPosition - when lenient $ - optional (char '$' >> parseNote ErrorC 1066 "Don't use $ on the left side of assignments.") - variable <- readVariableName - when lenient $ - optional (readNormalDollar >> parseNoteAt pos ErrorC - 1067 "For indirection, use arrays, declare \"var$n=value\", or (for sh) read/eval.") - indices <- many readArrayIndex - hasLeftSpace <- fmap (not . null) spacing - pos <- getPosition - id <- endSpan start - op <- readAssignmentOp +readAssignmentWordExt lenient = called "variable assignment" $ do + -- Parse up to and including the = in a 'try' + (id, variable, op, indices) <- try $ do + start <- startSpan + pos <- getPosition + leadingDollarPos <- + if lenient + then optionMaybe $ getSpanPositionsFor (char '$') + else return Nothing + variable <- readVariableName + middleDollarPos <- + if lenient + then optionMaybe $ getSpanPositionsFor readNormalDollar + else return Nothing + indices <- many readArrayIndex + hasLeftSpace <- fmap (not . null) spacing + opStart <- getPosition + id <- endSpan start + op <- readAssignmentOp + opEnd <- getPosition + + when (isJust leadingDollarPos || isJust middleDollarPos || hasLeftSpace) $ do + sequence_ $ do + (l, r) <- leadingDollarPos + return $ parseProblemAtWithEnd l r ErrorC 1066 "Don't use $ on the left side of assignments." + sequence_ $ do + (l, r) <- middleDollarPos + return $ parseProblemAtWithEnd l r ErrorC 1067 "For indirection, use arrays, declare \"var$n=value\", or (for sh) read/eval." + when hasLeftSpace $ do + parseProblemAtWithEnd opStart opEnd ErrorC 1068 $ + "Don't put spaces around the " + ++ (if op == Append + then "+= when appending" + else "= in assignments") + ++ " (or quote to make it literal)." + + -- Fail so that this is not parsed as an assignment. + fail "" + -- At this point we know for sure. + return (id, variable, op, indices) + + rightPosStart <- getPosition hasRightSpace <- fmap (not . null) spacing + rightPosEnd <- getPosition isEndOfCommand <- fmap isJust $ optionMaybe (try . lookAhead $ (void (oneOf "\r\n;&|)") <|> eof)) - if not hasLeftSpace && (hasRightSpace || isEndOfCommand) + + if hasRightSpace || isEndOfCommand then do - when (variable /= "IFS" && hasRightSpace && not isEndOfCommand) $ - parseNoteAt pos WarningC 1007 + when (variable /= "IFS" && hasRightSpace && not isEndOfCommand) $ do + parseProblemAtWithEnd rightPosStart rightPosEnd WarningC 1007 "Remove space after = if trying to assign a value (for empty string, use var='' ... )." value <- readEmptyLiteral return $ T_Assignment id op variable indices value else do - when (hasLeftSpace || hasRightSpace) $ - parseNoteAt pos ErrorC 1068 $ - "Don't put spaces around the " - ++ (if op == Append - then "+= when appending" - else "= in assignments") - ++ " (or quote to make it literal)." + optional $ do + lookAhead $ char '=' + parseProblem ErrorC 1097 "Unexpected ==. For assignment, use =. For comparison, use [/[[. Or quote for literal string." + value <- readArray <|> readNormalWord spacing return $ T_Assignment id op variable indices value where readAssignmentOp = do - pos <- getPosition - unexpecting "" $ string "===" + -- This is probably some kind of ascii art border + unexpecting "===" (string "===") choice [ string "+=" >> return Append, - do - try (string "==") - parseProblemAt pos ErrorC 1097 - "Unexpected ==. For assignment, use =. For comparison, use [/[[." - return Assign, - string "=" >> return Assign ] From 43191fa71de1a67e04298db2cda14169e43d50c2 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 1 Sep 2020 14:19:28 -0700 Subject: [PATCH 387/763] Suppress SC2035 for echo * and printf * (fixes #2036) --- src/ShellCheck/Analytics.hs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index ffcf9cf..e0d33bb 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2320,8 +2320,11 @@ prop_checkGlobsAsOptions1 = verify checkGlobsAsOptions "rm *.txt" prop_checkGlobsAsOptions2 = verify checkGlobsAsOptions "ls ??.*" prop_checkGlobsAsOptions3 = verifyNot checkGlobsAsOptions "rm -- *.txt" prop_checkGlobsAsOptions4 = verifyNot checkGlobsAsOptions "*.txt" -checkGlobsAsOptions _ (T_SimpleCommand _ _ args) = - mapM_ check $ takeWhile (not . isEndOfArgs) (drop 1 args) +prop_checkGlobsAsOptions5 = verifyNot checkGlobsAsOptions "echo 'Files:' *.txt" +prop_checkGlobsAsOptions6 = verifyNot checkGlobsAsOptions "printf '%s\\n' *" +checkGlobsAsOptions _ cmd@(T_SimpleCommand _ _ args) = + unless ((fromMaybe "" $ getCommandBasename cmd) `elem` ["echo", "printf"]) $ + mapM_ check $ takeWhile (not . isEndOfArgs) (drop 1 args) where check v@(T_NormalWord _ (T_Glob id s:_)) | s == "*" || s == "?" = info id 2035 "Use ./*glob* or -- *glob* so names with dashes won't become options." From 58783ab3cc11ff30466f185a88b1ebb2f1bf19f5 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 1 Sep 2020 16:22:15 -0700 Subject: [PATCH 388/763] Allow specifying ranges in disable directives --- CHANGELOG.md | 1 + shellcheck.1.md | 3 ++- src/ShellCheck/AST.hs | 2 +- src/ShellCheck/ASTLib.hs | 2 +- src/ShellCheck/Parser.hs | 11 ++++++++--- 5 files changed, 13 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5db445b..53f63da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ## Git ### Added +- `disable` directives can now be a range, e.g. `disable=SC3000-SC4000` - SC2259/SC2260: Warn when redirections override pipes - SC2261: Warn about multiple competing redirections - SC2262/SC2263: Warn about aliases declared and used in the same parsing unit diff --git a/shellcheck.1.md b/shellcheck.1.md index 6d2a732..d038df2 100644 --- a/shellcheck.1.md +++ b/shellcheck.1.md @@ -232,7 +232,8 @@ Valid keys are: **disable** : Disables a comma separated list of error codes for the following command. The command can be a simple command like `echo foo`, or a compound command - like a function definition, subshell block or loop. + like a function definition, subshell block or loop. A range can be + be specified with a dash, e.g. `disable=SC3000-SC4000` to exclude 3xxx. **enable** : Enable an optional check by name, as listed with **--list-optional**. diff --git a/src/ShellCheck/AST.hs b/src/ShellCheck/AST.hs index d4151fe..52b6e17 100644 --- a/src/ShellCheck/AST.hs +++ b/src/ShellCheck/AST.hs @@ -145,7 +145,7 @@ data InnerToken t = deriving (Show, Eq, Functor, Foldable, Traversable) data Annotation = - DisableComment Integer + DisableComment Integer Integer -- [from, to) | EnableComment String | SourceOverride String | ShellOverride String diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 29feb1e..0052ef5 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -567,5 +567,5 @@ isAnnotationIgnoringCode code t = T_Annotation _ anns _ -> any hasNum anns _ -> False where - hasNum (DisableComment ts) = code == ts + hasNum (DisableComment from to) = code >= from && code < to hasNum _ = False diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 4806b18..87efa25 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -270,7 +270,7 @@ contextItemDisablesCode alsoCheckSourced code = disabling alsoCheckSourced ContextAnnotation list -> any disabling' list ContextSource _ -> not $ checkSourced _ -> False - disabling' (DisableComment n) = code == n + disabling' (DisableComment n m) = code >= n && code < m disabling' _ = False @@ -973,6 +973,7 @@ prop_readAnnotation3 = isOk readAnnotation "# shellcheck disable=SC1234 source=/ prop_readAnnotation4 = isWarning readAnnotation "# shellcheck cats=dogs disable=SC1234\n" prop_readAnnotation5 = isOk readAnnotation "# shellcheck disable=SC2002 # All cats are precious\n" prop_readAnnotation6 = isOk readAnnotation "# shellcheck disable=SC1234 # shellcheck foo=bar\n" +prop_readAnnotation7 = isOk readAnnotation "# shellcheck disable=SC1000,SC2000-SC3000,SC1001\n" readAnnotation = called "shellcheck directive" $ do try readAnnotationPrefix many1 linewhitespace @@ -993,12 +994,16 @@ readAnnotationWithoutPrefix = do key <- many1 (letter <|> char '-') char '=' <|> fail "Expected '=' after directive key" annotations <- case key of - "disable" -> readCode `sepBy` char ',' + "disable" -> readRange `sepBy` char ',' where + readRange = do + from <- readCode + to <- choice [ char '-' *> readCode, return $ from+1 ] + return $ DisableComment from to readCode = do optional $ string "SC" int <- many1 digit - return $ DisableComment (read int) + return $ read int "enable" -> readName `sepBy` char ',' where From cfd68ee0c2ebfd0ab08a1d4bf628162b454dc207 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 1 Sep 2020 16:48:14 -0700 Subject: [PATCH 389/763] Give each sh/dash compatibility warning its own SC3xxx error code --- CHANGELOG.md | 1 + nextnumber | 2 +- src/ShellCheck/Checker.hs | 2 +- src/ShellCheck/Checks/ShellSupport.hs | 115 +++++++++++++------------- 4 files changed, 62 insertions(+), 58 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 53f63da..d4b9145 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ ### Changed - Assignments are now parsed to spec, without leniency for leading $ or spaces +- POSIX/dash unsupported feature warnings now have individual SC3xxx codes - SC1090: A leading `$x/` or `$(x)/` is now treated as `./` when locating files - SC2154: Variables appearing in -z/-n tests are no longer considered unassigned diff --git a/nextnumber b/nextnumber index 0c9c7b1..3bdf10a 100755 --- a/nextnumber +++ b/nextnumber @@ -6,7 +6,7 @@ then exit 1 fi -for i in 1 2 +for i in 1 2 3 do last=$(grep -hv "^prop" ./**/*.hs | grep -Ewo "${i}[0-9]{3}" | sort -n | tail -n 1) echo "Next ${i}xxx: $((last+1))" diff --git a/src/ShellCheck/Checker.hs b/src/ShellCheck/Checker.hs index 673a116..d81d664 100644 --- a/src/ShellCheck/Checker.hs +++ b/src/ShellCheck/Checker.hs @@ -276,7 +276,7 @@ prop_filewideAnnotation8 = null $ check "# Disable $? warning\n#shellcheck disable=SC2181\n# Disable quoting warning\n#shellcheck disable=2086\ntrue\n[ $? == 0 ] && echo $1" prop_sourcePartOfOriginalScript = -- #1181: -x disabled posix warning for 'source' - 2039 `elem` checkWithIncludes [("./saywhat.sh", "echo foo")] "#!/bin/sh\nsource ./saywhat.sh" + 3046 `elem` checkWithIncludes [("./saywhat.sh", "echo foo")] "#!/bin/sh\nsource ./saywhat.sh" prop_spinBug1413 = null $ check "fun() {\n# shellcheck disable=SC2188\n> /dev/null\n}\n" diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 788323d..8b8e095 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -188,102 +188,102 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do kludge params = bashism where isDash = shellType params == Dash - warnMsg id s = + warnMsg id code s = if isDash - then err id 2169 $ "In dash, " ++ s ++ " not supported." - else warn id 2039 $ "In POSIX sh, " ++ s ++ " undefined." + then err id code $ "In dash, " ++ s ++ " not supported." + else warn id code $ "In POSIX sh, " ++ s ++ " undefined." - bashism (T_ProcSub id _ _) = warnMsg id "process substitution is" - bashism (T_Extglob id _ _) = warnMsg id "extglob is" - bashism (T_DollarSingleQuoted id _) = warnMsg id "$'..' is" - bashism (T_DollarDoubleQuoted id _) = warnMsg id "$\"..\" is" - bashism (T_ForArithmetic id _ _ _ _) = warnMsg id "arithmetic for loops are" - bashism (T_Arithmetic id _) = warnMsg id "standalone ((..)) is" - bashism (T_DollarBracket id _) = warnMsg id "$[..] in place of $((..)) is" - bashism (T_SelectIn id _ _ _) = warnMsg id "select loops are" - bashism (T_BraceExpansion id _) = warnMsg id "brace expansion is" - bashism (T_Condition id DoubleBracket _) = warnMsg id "[[ ]] is" - bashism (T_HereString id _) = warnMsg id "here-strings are" + bashism (T_ProcSub id _ _) = warnMsg id 3001 "process substitution is" + bashism (T_Extglob id _ _) = warnMsg id 3002 "extglob is" + bashism (T_DollarSingleQuoted id _) = warnMsg id 3003 "$'..' is" + bashism (T_DollarDoubleQuoted id _) = warnMsg id 3004 "$\"..\" is" + bashism (T_ForArithmetic id _ _ _ _) = warnMsg id 3005 "arithmetic for loops are" + bashism (T_Arithmetic id _) = warnMsg id 3006 "standalone ((..)) is" + bashism (T_DollarBracket id _) = warnMsg id 3007 "$[..] in place of $((..)) is" + bashism (T_SelectIn id _ _ _) = warnMsg id 3008 "select loops are" + bashism (T_BraceExpansion id _) = warnMsg id 3009 "brace expansion is" + bashism (T_Condition id DoubleBracket _) = warnMsg id 3010 "[[ ]] is" + bashism (T_HereString id _) = warnMsg id 3011 "here-strings are" bashism (TC_Binary id SingleBracket op _ _) | op `elem` [ "<", ">", "\\<", "\\>", "<=", ">=", "\\<=", "\\>="] = - unless isDash $ warnMsg id $ "lexicographical " ++ op ++ " is" + unless isDash $ warnMsg id 3012 $ "lexicographical " ++ op ++ " is" bashism (TC_Binary id SingleBracket op _ _) | op `elem` [ "-ot", "-nt", "-ef" ] = - unless isDash $ warnMsg id $ op ++ " is" + unless isDash $ warnMsg id 3013 $ op ++ " is" bashism (TC_Binary id SingleBracket "==" _ _) = - warnMsg id "== in place of = is" + warnMsg id 3014 "== in place of = is" bashism (TC_Binary id SingleBracket "=~" _ _) = - warnMsg id "=~ regex matching is" + warnMsg id 3015 "=~ regex matching is" bashism (TC_Unary id SingleBracket "-v" _) = - warnMsg id "unary -v (in place of [ -n \"${var+x}\" ]) is" + warnMsg id 3016 "unary -v (in place of [ -n \"${var+x}\" ]) is" bashism (TC_Unary id _ "-a" _) = - warnMsg id "unary -a in place of -e is" + warnMsg id 3017 "unary -a in place of -e is" bashism (TA_Unary id op _) | op `elem` [ "|++", "|--", "++|", "--|"] = - warnMsg id $ filter (/= '|') op ++ " is" - bashism (TA_Binary id "**" _ _) = warnMsg id "exponentials are" - bashism (T_FdRedirect id "&" (T_IoFile _ (T_Greater _) _)) = warnMsg id "&> is" - bashism (T_FdRedirect id "" (T_IoFile _ (T_GREATAND _) _)) = warnMsg id ">& is" - bashism (T_FdRedirect id ('{':_) _) = warnMsg id "named file descriptors are" + warnMsg id 3018 $ filter (/= '|') op ++ " is" + bashism (TA_Binary id "**" _ _) = warnMsg id 3019 "exponentials are" + bashism (T_FdRedirect id "&" (T_IoFile _ (T_Greater _) _)) = warnMsg id 3020 "&> is" + bashism (T_FdRedirect id "" (T_IoFile _ (T_GREATAND _) _)) = warnMsg id 3021 ">& is" + bashism (T_FdRedirect id ('{':_) _) = warnMsg id 3022 "named file descriptors are" bashism (T_FdRedirect id num _) - | all isDigit num && length num > 1 = warnMsg id "FDs outside 0-9 are" + | all isDigit num && length num > 1 = warnMsg id 3023 "FDs outside 0-9 are" bashism (T_Assignment id Append _ _ _) = - warnMsg id "+= is" + warnMsg id 3024 "+= is" bashism (T_IoFile id _ word) | isNetworked = - warnMsg id "/dev/{tcp,udp} is" + warnMsg id 3025 "/dev/{tcp,udp} is" where file = onlyLiteralString word isNetworked = any (`isPrefixOf` file) ["/dev/tcp", "/dev/udp"] bashism (T_Glob id str) | "[^" `isInfixOf` str = - warnMsg id "^ in place of ! in glob bracket expressions is" + warnMsg id 3026 "^ in place of ! in glob bracket expressions is" bashism t@(TA_Variable id str _) | isBashVariable str = - warnMsg id $ str ++ " is" + warnMsg id 3027 $ str ++ " is" bashism t@(T_DollarBraced id _ token) = do mapM_ check expansion when (isBashVariable var) $ - warnMsg id $ var ++ " is" + warnMsg id 3028 $ var ++ " is" where str = concat $ oversimplify token var = getBracedReference str check (regex, feature) = - when (isJust $ matchRegex regex str) $ warnMsg id feature + when (isJust $ matchRegex regex str) $ warnMsg id 3053 feature bashism t@(T_Pipe id "|&") = - warnMsg id "|& in place of 2>&1 | is" + warnMsg id 3029 "|& in place of 2>&1 | is" bashism (T_Array id _) = - warnMsg id "arrays are" + warnMsg id 3030 "arrays are" bashism (T_IoFile id _ t) | isGlob t = - warnMsg id "redirecting to/from globs is" + warnMsg id 3031 "redirecting to/from globs is" bashism (T_CoProc id _ _) = - warnMsg id "coproc is" + warnMsg id 3032 "coproc is" bashism (T_Function id _ _ str _) | not (isVariableName str) = - warnMsg id "naming functions outside [a-zA-Z_][a-zA-Z0-9_]* is" + warnMsg id 3033 "naming functions outside [a-zA-Z_][a-zA-Z0-9_]* is" bashism (T_DollarExpansion id [x]) | isOnlyRedirection x = - warnMsg id "$( do checkOptions (flag@(fid,flag') : opt@(oid,opt') : rest) | flag' `matches` oFlagRegex = do when (opt' `notElem` longOptions) $ - warnMsg oid $ "set option " <> opt' <> " is" + warnMsg oid 3040 $ "set option " <> opt' <> " is" checkFlags (flag:rest) | otherwise = checkFlags (flag:opt:rest) checkOptions (flag:rest) = checkFlags (flag:rest) @@ -314,10 +314,10 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do unless (flag' `matches` validFlagsRegex) $ forM_ (tail flag') $ \letter -> when (letter `notElem` optionsSet) $ - warnMsg fid $ "set flag " <> ('-':letter:" is") + warnMsg fid 3041 $ "set flag " <> ('-':letter:" is") checkOptions rest | beginsWithDoubleDash flag' = do - warnMsg fid $ "set flag " <> flag' <> " is" + warnMsg fid 3042 $ "set flag " <> flag' <> " is" checkOptions rest -- Either a word that doesn't start with a dash, or simply '--', -- so stop checking. @@ -339,16 +339,19 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do let name = fromMaybe "" $ getCommandName t flags = getLeadingFlags t in do + when (name == "local" && not isDash) $ + -- This is so commonly accepted that we'll make it a special case + warnMsg id 3043 $ "'local' is" when (name `elem` unsupportedCommands) $ - warnMsg id $ "'" ++ name ++ "' is" + warnMsg id 3044 $ "'" ++ name ++ "' is" sequence_ $ do allowed' <- Map.lookup name allowedFlags allowed <- allowed' (word, flag) <- find (\x -> (not . null . snd $ x) && snd x `notElem` allowed) flags - return . warnMsg (getId word) $ name ++ " -" ++ flag ++ " is" + return . warnMsg (getId word) 3045 $ name ++ " -" ++ flag ++ " is" - when (name == "source") $ warnMsg id "'source' in place of '.' is" + when (name == "source") $ warnMsg id 3046 "'source' in place of '.' is" when (name == "trap") $ let check token = sequence_ $ do @@ -356,12 +359,12 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do let upper = map toUpper str return $ do when (upper `elem` ["ERR", "DEBUG", "RETURN"]) $ - warnMsg (getId token) $ "trapping " ++ str ++ " is" + warnMsg (getId token) 3047 $ "trapping " ++ str ++ " is" when ("SIG" `isPrefixOf` upper) $ - warnMsg (getId token) + warnMsg (getId token) 3048 "prefixing signal names with 'SIG' is" when (not isDash && upper /= str) $ - warnMsg (getId token) + warnMsg (getId token) 3049 "using lower/mixed case for signal names is" in mapM_ check (drop 1 rest) @@ -370,13 +373,13 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do format <- rest !!! 0 -- flags are covered by allowedFlags let literal = onlyLiteralString format guard $ "%q" `isInfixOf` literal - return $ warnMsg (getId format) "printf %q is" + return $ warnMsg (getId format) 3050 "printf %q is" where unsupportedCommands = [ "let", "caller", "builtin", "complete", "compgen", "declare", "dirs", "disown", "enable", "mapfile", "readarray", "pushd", "popd", "shopt", "suspend", "typeset" - ] ++ if not isDash then ["local"] else [] + ] allowedFlags = Map.fromList [ ("cd", Just ["L", "P"]), ("exec", Just []), @@ -394,9 +397,9 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do ("wait", Just []) ] bashism t@(T_SourceCommand id src _) - | getCommandName src == Just "source" = warnMsg id "'source' in place of '.' is" + | getCommandName src == Just "source" = warnMsg id 3051 "'source' in place of '.' is" bashism (TA_Expansion _ (T_Literal id str : _)) - | str `matches` radix = warnMsg id "arithmetic base conversion is" + | str `matches` radix = warnMsg id 3052 "arithmetic base conversion is" where radix = mkRegex "^[0-9]+#" bashism _ = return () From c4cc2debb78b4448d37a672735f0c20a76cd7d51 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 7 Sep 2020 21:05:49 -0700 Subject: [PATCH 390/763] Improve compatibility checks --- src/ShellCheck/Checks/ShellSupport.hs | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 8b8e095..c4db9ce 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -180,6 +180,7 @@ prop_checkBashisms95 = verify checkBashisms "#!/bin/sh\necho $_" prop_checkBashisms96 = verifyNot checkBashisms "#!/bin/dash\necho $_" prop_checkBashisms97 = verify checkBashisms "#!/bin/sh\necho ${var,}" prop_checkBashisms98 = verify checkBashisms "#!/bin/sh\necho ${var^^}" +prop_checkBashisms99 = verifyNot checkBashisms "#!/bin/dash\necho [^f]oo" checkBashisms = ForShell [Sh, Dash] $ \t -> do params <- ask kludge params t @@ -234,11 +235,11 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do where file = onlyLiteralString word isNetworked = any (`isPrefixOf` file) ["/dev/tcp", "/dev/udp"] - bashism (T_Glob id str) | "[^" `isInfixOf` str = + bashism (T_Glob id str) | not isDash && "[^" `isInfixOf` str = warnMsg id 3026 "^ in place of ! in glob bracket expressions is" bashism t@(TA_Variable id str _) | isBashVariable str = - warnMsg id 3027 $ str ++ " is" + warnMsg id 3028 $ str ++ " is" bashism t@(T_DollarBraced id _ token) = do mapM_ check expansion @@ -247,8 +248,8 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do where str = concat $ oversimplify token var = getBracedReference str - check (regex, feature) = - when (isJust $ matchRegex regex str) $ warnMsg id 3053 feature + check (regex, code, feature) = + when (isJust $ matchRegex regex str) $ warnMsg id code feature bashism t@(T_Pipe id "|&") = warnMsg id 3029 "|& in place of 2>&1 | is" @@ -406,14 +407,14 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do varChars="_0-9a-zA-Z" expansion = let re = mkRegex in [ - (re $ "^![" ++ varChars ++ "]", "indirect expansion is"), - (re $ "^[" ++ varChars ++ "]+\\[.*\\]$", "array references are"), - (re $ "^![" ++ varChars ++ "]+\\[[*@]]$", "array key expansion is"), - (re $ "^![" ++ varChars ++ "]+[*@]$", "name matching prefixes are"), - (re $ "^[" ++ varChars ++ "*@]+:[^-=?+]", "string indexing is"), - (re $ "^([*@][%#]|#[@*])", "string operations on $@/$* are"), - (re $ "^[" ++ varChars ++ "*@]+(\\[.*\\])?[,^]", "case modification is"), - (re $ "^[" ++ varChars ++ "*@]+(\\[.*\\])?/", "string replacement is") + (re $ "^![" ++ varChars ++ "]", 3053, "indirect expansion is"), + (re $ "^[" ++ varChars ++ "]+\\[.*\\]$", 3054, "array references are"), + (re $ "^![" ++ varChars ++ "]+\\[[*@]]$", 3055, "array key expansion is"), + (re $ "^![" ++ varChars ++ "]+[*@]$", 3056, "name matching prefixes are"), + (re $ "^[" ++ varChars ++ "*@]+:[^-=?+]", 3057, "string indexing is"), + (re $ "^([*@][%#]|#[@*])", 3058, "string operations on $@/$* are"), + (re $ "^[" ++ varChars ++ "*@]+(\\[.*\\])?[,^]", 3059, "case modification is"), + (re $ "^[" ++ varChars ++ "*@]+(\\[.*\\])?/", 3060, "string replacement is") ] bashVars = [ "OSTYPE", "MACHTYPE", "HOSTTYPE", "HOSTNAME", From 218deb6d01ec327fdf91d8d3a00f1ecccfc9df44 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 8 Sep 2020 19:30:13 -0700 Subject: [PATCH 391/763] Update SC2091/SC2092 message and ignore in quotes. --- src/ShellCheck/Analytics.hs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index e0d33bb..c39f8c8 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1692,19 +1692,17 @@ checkSpuriousExec _ = doLists prop_checkSpuriousExpansion1 = verify checkSpuriousExpansion "if $(true); then true; fi" -prop_checkSpuriousExpansion2 = verify checkSpuriousExpansion "while \"$(cmd)\"; do :; done" prop_checkSpuriousExpansion3 = verifyNot checkSpuriousExpansion "$(cmd) --flag1 --flag2" prop_checkSpuriousExpansion4 = verify checkSpuriousExpansion "$((i++))" checkSpuriousExpansion _ (T_SimpleCommand _ _ [T_NormalWord _ [word]]) = check word where check word = case word of T_DollarExpansion id _ -> - warn id 2091 "Remove surrounding $() to avoid executing output." + warn id 2091 "Remove surrounding $() to avoid executing output (or use eval if intentional)." T_Backticked id _ -> - warn id 2092 "Remove backticks to avoid executing output." + warn id 2092 "Remove backticks to avoid executing output (or use eval if intentional)." T_DollarArithmetic id _ -> err id 2084 "Remove '$' or use '_=$((expr))' to avoid executing output." - T_DoubleQuoted id [subword] -> check subword _ -> return () checkSpuriousExpansion _ _ = return () From 8d999265541e28a1fa6f73364d70cc4cbcc70559 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 18 Oct 2020 15:15:31 -0700 Subject: [PATCH 392/763] Recognize `local -x` similarly to `export` (fixes #2069) --- src/ShellCheck/Analytics.hs | 1 + src/ShellCheck/AnalyzerLib.hs | 3 +++ 2 files changed, 4 insertions(+) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index c39f8c8..e5287f8 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2134,6 +2134,7 @@ prop_checkUnused13= verifyNotTree checkUnusedAssignments "x=(1); (( x[0] ))" prop_checkUnused14= verifyNotTree checkUnusedAssignments "x=(1); n=0; echo ${x[n]}" prop_checkUnused15= verifyNotTree checkUnusedAssignments "x=(1); n=0; (( x[n] ))" prop_checkUnused16= verifyNotTree checkUnusedAssignments "foo=5; declare -x foo" +prop_checkUnused16b= verifyNotTree checkUnusedAssignments "f() { local -x foo; foo=42; bar; }; f" prop_checkUnused17= verifyNotTree checkUnusedAssignments "read -i 'foo' -e -p 'Input: ' bar; $bar;" prop_checkUnused18= verifyNotTree checkUnusedAssignments "a=1; arr=( [$a]=42 ); echo \"${arr[@]}\"" prop_checkUnused19= verifyNotTree checkUnusedAssignments "a=1; let b=a+1; echo $b" diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index c9954d1..724857c 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -547,6 +547,9 @@ getReferencedVariableCommand base@(T_SimpleCommand _ _ (T_NormalWord _ (T_Litera (not $ any (`elem` flags) ["f", "F"]) then concatMap getReference rest else [] + "local" -> if "x" `elem` flags + then concatMap getReference rest + else [] "trap" -> case rest of head:_ -> map (\x -> (base, head, x)) $ getVariablesFromLiteralToken head From f100c2939e7fd50975b20d3c7e7cd9e635b1b428 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 18 Oct 2020 20:36:48 -0700 Subject: [PATCH 393/763] Rewrite getopts style option parser --- src/ShellCheck/ASTLib.hs | 103 ++++++++++++++++++++++++------ src/ShellCheck/Analytics.hs | 6 +- src/ShellCheck/AnalyzerLib.hs | 4 +- src/ShellCheck/Checks/Commands.hs | 39 +++++++++-- 4 files changed, 120 insertions(+), 32 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 0052ef5..a09bf38 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -30,6 +30,8 @@ import Data.List import Data.Maybe import qualified Data.Map as Map +arguments (T_SimpleCommand _ _ (cmd:args)) = args + -- Is this a type of loop? isLoop t = case t of T_WhileExpression {} -> True @@ -135,32 +137,91 @@ isUnquotedFlag token = fromMaybe False $ do str <- getLeadingUnquotedString token return $ "-" `isPrefixOf` str --- getGnuOpts "erd:u:" will parse a SimpleCommand like --- read -re -d : -u 3 bar +-- getGnuOpts "erd:u:" will parse a list of arguments tokens like `read` +-- -re -d : -u 3 bar -- into --- Just [("r", -re), ("e", -re), ("d", :), ("u", 3), ("", bar)] --- where flags with arguments map to arguments, while others map to themselves. --- Any unrecognized flag will result in Nothing. -getGnuOpts str t = getOpts str $ getAllFlags t -getBsdOpts str t = getOpts str $ getLeadingFlags t -getOpts :: String -> [(Token, String)] -> Maybe [(String, Token)] -getOpts string flags = process flags +-- Just [("r", (-re, -re)), ("e", (-re, -re)), ("d", (-d,:)), ("u", (-u,3)), ("", (bar,bar))] +-- +-- Each string flag maps to a tuple of (flag, argument), where argument=flag if it +-- doesn't take a specific one. +-- +-- Any unrecognized flag will result in Nothing. The exception is if arbitraryLongOpts +-- is set, in which case --anything will map to "anything". +getGnuOpts :: String -> [Token] -> Maybe [(String, (Token, Token))] +getGnuOpts str args = getOpts (True, False) str [] args + +-- As above, except the first non-arg string will treat the rest as arguments +getBsdOpts :: String -> [Token] -> Maybe [(String, (Token, Token))] +getBsdOpts str args = getOpts (False, False) str [] args + +-- Tests for this are in Commands.hs where it's more frequently used +getOpts :: + -- Behavioral config: gnu style, allow arbitrary long options + (Bool, Bool) + -- A getopts style string + -> String + -- List of long options and whether they take arguments + -> [(String, Bool)] + -- List of arguments (excluding command) + -> [Token] + -- List of flags to tuple of (optionToken, valueToken) + -> Maybe [(String, (Token, Token))] + +getOpts (gnu, arbitraryLongOpts) string longopts args = process args where flagList (c:':':rest) = ([c], True) : flagList rest flagList (c:rest) = ([c], False) : flagList rest - flagList [] = [] + flagList [] = longopts flagMap = Map.fromList $ ("", False) : flagList string process [] = return [] - process ((token1, flag):rest1) = do - takesArg <- Map.lookup flag flagMap - (token, rest) <- if takesArg - then case rest1 of - (token2, ""):rest2 -> return (token2, rest2) - _ -> fail "takesArg without valid arg" - else return (token1, rest1) - more <- process rest - return $ (flag, token) : more + process (token:rest) = do + case getLiteralStringDef "\0" token of + '-':'-':[] -> return $ listToArgs rest + '-':'-':word -> do + let (name, arg) = span (/= '=') word + needsArg <- + if arbitraryLongOpts + then return $ Map.findWithDefault False name flagMap + else Map.lookup name flagMap + + if needsArg && null arg + then + case rest of + (arg:rest2) -> do + more <- process rest2 + return $ (name, (token, arg)) : more + _ -> fail "Missing arg" + else do + more <- process rest + -- Consider splitting up token to get arg + return $ (name, (token, token)) : more + '-':opts -> shortToOpts opts token rest + arg -> + if gnu + then do + more <- process rest + return $ ("", (token, token)):more + else return $ listToArgs (token:rest) + + shortToOpts opts token args = + case opts of + c:rest -> do + needsArg <- Map.lookup [c] flagMap + case () of + _ | needsArg && null rest -> do + (next:restArgs) <- return args + more <- process restArgs + return $ ([c], (token, next)):more + _ | needsArg -> do + more <- process args + return $ ([c], (token, token)):more + _ -> do + more <- shortToOpts rest token args + return $ ([c], (token, token)):more + [] -> process args + + listToArgs = map (\x -> ("", (x, x))) -- Is this an expansion of multiple items of an array? isArrayExpansion (T_DollarBraced _ _ l) = @@ -362,8 +423,8 @@ getCommandNameAndToken direct t = fromMaybe (Nothing, t) $ do "builtin" -> firstArg "command" -> firstArg "exec" -> do - opts <- getBsdOpts "cla:" cmd - (_, t) <- listToMaybe $ filter (null . fst) opts + opts <- getBsdOpts "cla:" args + (_, (t, _)) <- listToMaybe $ filter (null . fst) opts return t _ -> fail "" diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index e5287f8..cf7bcb1 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2925,8 +2925,8 @@ checkReadWithoutR _ t@T_SimpleCommand {} | t `isUnqualifiedCommand` "read" where flags = getAllFlags t has_t0 = Just "0" == do - parsed <- getOpts flagsForRead flags - t <- lookup "t" parsed + parsed <- getGnuOpts flagsForRead $ arguments t + (_, t) <- lookup "t" parsed getLiteralString t checkReadWithoutR _ _ = return () @@ -3383,7 +3383,7 @@ checkPipeToNowhere params t = commandSpecificException name cmd = case name of - "du" -> any (`elem` ["exclude-from", "files0-from"]) $ lt $ map snd $ getAllFlags cmd + "du" -> any (`elem` ["exclude-from", "files0-from"]) $ map snd $ getAllFlags cmd _ -> False warnAboutDupes (n, list@(_:_:_)) = diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 724857c..0f40b3b 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -678,13 +678,13 @@ getModifiedVariableCommand base@(T_SimpleCommand id cmdPrefix (T_NormalWord _ (T where parseArgs :: Maybe (Token, Token, String, DataType) parseArgs = do - args <- getGnuOpts "d:n:O:s:u:C:c:t" base + args <- getGnuOpts "d:n:O:s:u:C:c:t" rest let names = map snd $ filter (\(x,y) -> null x) args if null names then return (base, base, "MAPFILE", DataArray SourceExternal) else do - first <- listToMaybe names + (_, first) <- listToMaybe names name <- getLiteralString first guard $ isVariableName name return (base, first, name, DataArray SourceExternal) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index b4e8b3e..9af7748 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -53,8 +53,6 @@ verify :: CommandCheck -> String -> Bool verify f s = producesComments (getChecker [f]) s == Just True verifyNot f s = producesComments (getChecker [f]) s == Just False -arguments (T_SimpleCommand _ _ (cmd:args)) = args - commandChecks :: [CommandCheck] commandChecks = [ checkTr @@ -116,6 +114,35 @@ prop_verifyOptionalExamples = all check optionalCommandChecks verify check (cdPositive desc) && verifyNot check (cdNegative desc) +-- Run a check against the getopt parser. If it fails, the lists are empty. +checkGetOpts str flags args f = + flags == actualFlags && args == actualArgs + where + toTokens = map (T_Literal (Id 0)) . words + opts = fromMaybe [] $ f (toTokens str) + actualFlags = filter (not . null) $ map fst opts + actualArgs = map (\(_, (_, x)) -> onlyLiteralString x) $ filter (null . fst) opts + +-- Short options +prop_checkGetOptsS1 = checkGetOpts "-f x" ["f"] [] $ getOpts (True, True) "f:" [] +prop_checkGetOptsS2 = checkGetOpts "-fx" ["f"] [] $ getOpts (True, True) "f:" [] +prop_checkGetOptsS3 = checkGetOpts "-f -x" ["f", "x"] [] $ getOpts (True, True) "fx" [] +prop_checkGetOptsS4 = checkGetOpts "-f -x" ["f"] [] $ getOpts (True, True) "f:" [] +prop_checkGetOptsS5 = checkGetOpts "-fx" [] [] $ getOpts (True, True) "fx:" [] + +-- Long options +prop_checkGetOptsL1 = checkGetOpts "--foo=bar baz" ["foo"] ["baz"] $ getOpts (True, False) "" [("foo", True)] +prop_checkGetOptsL2 = checkGetOpts "--foo bar baz" ["foo"] ["baz"] $ getOpts (True, False) "" [("foo", True)] +prop_checkGetOptsL3 = checkGetOpts "--foo baz" ["foo"] ["baz"] $ getOpts (True, True) "" [] +prop_checkGetOptsL4 = checkGetOpts "--foo baz" [] [] $ getOpts (True, False) "" [] + +-- Know when to terminate +prop_checkGetOptsT1 = checkGetOpts "-a x -b" ["a", "b"] ["x"] $ getOpts (True, True) "ab" [] +prop_checkGetOptsT2 = checkGetOpts "-a x -b" ["a"] ["x","-b"] $ getOpts (False, True) "ab" [] +prop_checkGetOptsT3 = checkGetOpts "-a -- -b" ["a"] ["-b"] $ getOpts (True, True) "ab" [] +prop_checkGetOptsT4 = checkGetOpts "-a -- -b" ["a", "b"] [] $ getOpts (True, True) "a:b" [] + + buildCommandMap :: [CommandCheck] -> Map.Map CommandName (Token -> Analysis) buildCommandMap = foldl' addCheck Map.empty where @@ -694,8 +721,8 @@ checkReadExpansions = CommandCheck (Exactly "read") check where options = getGnuOpts flagsForRead getVars cmd = fromMaybe [] $ do - opts <- options cmd - return [y | (x,y) <- opts, null x || x == "a"] + opts <- options $ arguments cmd + return [y | (x,(_, y)) <- opts, null x || x == "a"] check cmd = mapM_ warning $ getVars cmd warning t = sequence_ $ do @@ -1070,8 +1097,8 @@ prop_checkSudoArgs7 = verifyNot checkSudoArgs "sudo docker export foo" checkSudoArgs = CommandCheck (Basename "sudo") f where f t = sequence_ $ do - opts <- parseOpts t - let nonFlags = [x | ("",x) <- opts] + opts <- parseOpts $ arguments t + let nonFlags = [x | ("",(x, _)) <- opts] commandArg <- nonFlags !!! 0 command <- getLiteralString commandArg guard $ command `elem` builtins From 3104cec770a98d057fab3827f584c04486afbbb3 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 18 Oct 2020 22:08:40 -0700 Subject: [PATCH 394/763] SC2267: Warn about xargs -i (fixes #2058) --- CHANGELOG.md | 1 + src/ShellCheck/Checks/Commands.hs | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d4b9145..446303b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - SC2262/SC2263: Warn about aliases declared and used in the same parsing unit - SC2264: Warn about wrapper functions that blatantly recurse - SC2265/SC2266: Warn when using & or | with test statements +- SC2267: Warn when using xargs -i instead of -I ### Fixed - SC1072/SC1073 now respond to disable annotations, though ignoring parse errors diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 9af7748..aa4edd4 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -94,6 +94,7 @@ commandChecks = [ ,checkSudoArgs ,checkSourceArgs ,checkChmodDashr + ,checkXargsDashi ] optionalChecks = map fst optionalCommandChecks @@ -1129,5 +1130,18 @@ checkChmodDashr = CommandCheck (Basename "chmod") f guard $ flag == "-r" return $ warn (getId t) 2253 "Use -R to recurse, or explicitly a-r to remove read permissions." +prop_checkXargsDashi1 = verify checkXargsDashi "xargs -i{} echo {}" +prop_checkXargsDashi2 = verifyNot checkXargsDashi "xargs -I{} echo {}" +prop_checkXargsDashi3 = verifyNot checkXargsDashi "xargs sed -i -e foo" +prop_checkXargsDashi4 = verify checkXargsDashi "xargs -e sed -i foo" +prop_checkXargsDashi5 = verifyNot checkXargsDashi "xargs -x sed -i foo" +checkXargsDashi = CommandCheck (Basename "xargs") f + where + f t = sequence_ $ do + opts <- parseOpts $ arguments t + (option, value) <- lookup "i" opts + return $ info (getId option) 2267 "GNU xargs -i is deprecated in favor of -I{}" + parseOpts = getBsdOpts "0oprtxadR:S:J:L:l:n:P:s:e:E:i:I:" + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) From 256457c47af7c2b80f1ebb9cbd67fa8e47e789d5 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 18 Oct 2020 22:57:16 -0700 Subject: [PATCH 395/763] Use getopts parser to find 'read' arrays (fixes #2073) --- src/ShellCheck/Analytics.hs | 3 ++- src/ShellCheck/AnalyzerLib.hs | 22 ++++++++-------------- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index cf7bcb1..b5a8a16 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -851,7 +851,8 @@ prop_checkArrayWithoutIndex6 = verifyTree checkArrayWithoutIndex "echo $PIPESTAT prop_checkArrayWithoutIndex7 = verifyTree checkArrayWithoutIndex "a=(a b); a+=c" prop_checkArrayWithoutIndex8 = verifyTree checkArrayWithoutIndex "declare -a foo; foo=bar;" prop_checkArrayWithoutIndex9 = verifyTree checkArrayWithoutIndex "read -r -a arr <<< 'foo bar'; echo \"$arr\"" -prop_checkArrayWithoutIndex10= verifyTree checkArrayWithoutIndex "read -ra arr <<< 'foo bar'; echo \"$arr\"" +prop_checkArrayWithoutIndex10 = verifyTree checkArrayWithoutIndex "read -ra arr <<< 'foo bar'; echo \"$arr\"" +prop_checkArrayWithoutIndex11 = verifyNotTree checkArrayWithoutIndex "read -rpfoobar r; r=42" checkArrayWithoutIndex params _ = doVariableFlowAnalysis readF writeF defaultMap (variableFlow params) where diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 0f40b3b..dc081db 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -578,10 +578,14 @@ getModifiedVariableCommand base@(T_SimpleCommand id cmdPrefix (T_NormalWord _ (T "builtin" -> getModifiedVariableCommand $ T_SimpleCommand id cmdPrefix rest "read" -> - let params = map getLiteral rest - readArrayVars = getReadArrayVariables rest - in - catMaybes $ takeWhile isJust (reverse params) ++ readArrayVars + let fallback = catMaybes $ takeWhile isJust (reverse $ map getLiteral rest) + in fromMaybe fallback $ do + parsed <- getGnuOpts flagsForRead rest + case lookup "a" parsed of + Just (_, var) -> (:[]) <$> getLiteralArray var + Nothing -> return $ catMaybes $ + map (getLiteral . snd . snd) $ filter (null . fst) parsed + "getopts" -> case rest of opts:var:_ -> maybeToList $ getLiteral var @@ -698,16 +702,6 @@ getModifiedVariableCommand base@(T_SimpleCommand id cmdPrefix (T_NormalWord _ (T guard $ isVariableName name return (name, arg) - -- get all the array variables used in read, e.g. read -a arr - getReadArrayVariables args = - map (getLiteralArray . snd) - (filter (isArrayFlag . fst) (zip args (tail args))) - - isArrayFlag x = case getLiteralString x of - Just ('-':'-':_) -> False - Just ('-':str) -> 'a' `elem` str - _ -> False - -- get the FLAGS_ variable created by a shflags DEFINE_ call getFlagVariable (n:v:_) = do name <- getLiteralString n From 28d3279ba6a86c8dfc3f9d3c6262f58f57bc09dd Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 19 Oct 2020 18:25:18 -0700 Subject: [PATCH 396/763] Optional style warning about [ x$var = xval ] --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 42 +++++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 446303b..1e0eff4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - SC2264: Warn about wrapper functions that blatantly recurse - SC2265/SC2266: Warn when using & or | with test statements - SC2267: Warn when using xargs -i instead of -I +- Optional avoid-x-comparisons: Style warning SC2268 for `[ x$var = xval ]` ### Fixed - SC1072/SC1073 now respond to disable annotations, though ignoring parse errors diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index b5a8a16..f55b704 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -240,6 +240,13 @@ optionalTreeChecks = [ cdPositive = "echo $VAR", cdNegative = "VAR=hello; echo $VAR" }, checkUnassignedReferences' True) + + ,(newCheckDescription { + cdName = "avoid-x-comparisons", + cdDescription = "Warn about 'x'-prefix in comparisons", + cdPositive = "[ \"x$var\" = xval ]", + cdNegative = "[ \"$var\" = val ]" + }, nodeChecksToTreeCheck [checkComparisonWithLeadingX]) ] optionalCheckMap :: Map.Map String (Parameters -> Token -> [TokenComment]) @@ -3926,6 +3933,41 @@ checkBadTestAndOr params t = T_Annotation _ _ t -> isTest t _ -> False +prop_checkComparisonWithLeadingX1 = verify checkComparisonWithLeadingX "[ x$foo = xlol ]" +prop_checkComparisonWithLeadingX2 = verify checkComparisonWithLeadingX "test x$foo = xlol" +prop_checkComparisonWithLeadingX3 = verifyNot checkComparisonWithLeadingX "[ $foo = xbar ]" +prop_checkComparisonWithLeadingX4 = verifyNot checkComparisonWithLeadingX "test $foo = xbar" +prop_checkComparisonWithLeadingX5 = verify checkComparisonWithLeadingX "[ \"x$foo\" = 'xlol' ]" +prop_checkComparisonWithLeadingX6 = verify checkComparisonWithLeadingX "[ x\"$foo\" = x'lol' ]" +checkComparisonWithLeadingX params t = + case t of + TC_Binary id typ op lhs rhs | op == "=" || op == "==" -> + check lhs rhs + T_SimpleCommand _ _ [cmd, lhs, op, rhs] | + getLiteralString cmd == Just "test" && + getLiteralString op `elem` [Just "=", Just "=="] -> + check lhs rhs + _ -> return () + where + msg = "Avoid outdated x-prefix in comparisons as it no longer serves a purpose." + check lhs rhs = sequence_ $ do + l <- fixLeadingX lhs + r <- fixLeadingX rhs + return $ styleWithFix (getId lhs) 2268 msg $ fixWith [l, r] + + fixLeadingX token = + case getWordParts token of + T_Literal id ('x':_):_ -> + case token of + -- The side is a single, unquoted x, so we have to quote + T_NormalWord _ [T_Literal id "x"] -> + return $ replaceStart id params 1 "\"\"" + -- Otherwise we can just delete it + _ -> return $ replaceStart id params 1 "" + T_SingleQuoted id ('x':_):_ -> + -- Replace the single quote and x + return $ replaceStart id params 2 "'" + _ -> Nothing return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) From 619662adb63e027866a7511e695b87d8ca069654 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Fri, 23 Oct 2020 17:29:04 -0700 Subject: [PATCH 397/763] Add readonly to SC2155 This adds a warning for readonly masking the return value of function. This is mentioned in the wiki https://github.com/koalaman/shellcheck/wiki/SC2155#problematic-code-in-the-case-of-readonly but didn't actually produce a warning. Fixes https://github.com/koalaman/shellcheck/issues/1336 --- src/ShellCheck/Analytics.hs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index f55b704..c894075 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2902,11 +2902,14 @@ checkTestArgumentSplitting params t = prop_checkMaskedReturns1 = verify checkMaskedReturns "f() { local a=$(false); }" prop_checkMaskedReturns2 = verify checkMaskedReturns "declare a=$(false)" prop_checkMaskedReturns3 = verify checkMaskedReturns "declare a=\"`false`\"" -prop_checkMaskedReturns4 = verifyNot checkMaskedReturns "declare a; a=$(false)" -prop_checkMaskedReturns5 = verifyNot checkMaskedReturns "f() { local -r a=$(false); }" +prop_checkMaskedReturns4 = verify checkMaskedReturns "readonly a=$(false)" +prop_checkMaskedReturns5 = verify checkMaskedReturns "readonly a=\"`false`\"" +prop_checkMaskedReturns6 = verifyNot checkMaskedReturns "declare a; a=$(false)" +prop_checkMaskedReturns7 = verifyNot checkMaskedReturns "f() { local -r a=$(false); }" +prop_checkMaskedReturns8 = verifyNot checkMaskedReturns "a=$(false); readonly a" checkMaskedReturns _ t@(T_SimpleCommand id _ (cmd:rest)) = sequence_ $ do name <- getCommandName t - guard $ name `elem` ["declare", "export"] + guard $ name `elem` ["declare", "export", "readonly"] || name == "local" && "r" `notElem` map snd (getAllFlags t) return $ mapM_ checkArgs rest where From 4b0e5ca11933fb7735977c032ec80775ab7640aa Mon Sep 17 00:00:00 2001 From: Artur Klauser Date: Sun, 5 Apr 2020 21:11:35 +0200 Subject: [PATCH 398/763] Simplify .prepare-deploy Reduce amount of duplicated code. --- .prepare_deploy | 31 +++++-------------------------- 1 file changed, 5 insertions(+), 26 deletions(-) diff --git a/.prepare_deploy b/.prepare_deploy index 226c16d..a2ce361 100755 --- a/.prepare_deploy +++ b/.prepare_deploy @@ -27,35 +27,14 @@ do zip "${file%.*}.zip" README.txt LICENSE.txt "$file" done -for file in *.linux-x86_64 +for file in *.{linux,darwin}-* do base="${file%.*}" + ext="${file##*.}" + os="${ext%-*}" + arch="${ext##*-}" cp "$file" "shellcheck" - tar -cJf "$base.linux.x86_64.tar.xz" --transform="s:^:$base/:" README.txt LICENSE.txt shellcheck - rm "shellcheck" -done - -for file in *.linux-aarch64 -do - base="${file%.*}" - cp "$file" "shellcheck" - tar -cJf "$base.linux.aarch64.tar.xz" --transform="s:^:$base/:" README.txt LICENSE.txt shellcheck - rm "shellcheck" -done - -for file in *.linux-armv6hf -do - base="${file%.*}" - cp "$file" "shellcheck" - tar -cJf "$base.linux.armv6hf.tar.xz" --transform="s:^:$base/:" README.txt LICENSE.txt shellcheck - rm "shellcheck" -done - -for file in *.darwin-x86_64 -do - base="${file%.*}" - cp "$file" "shellcheck" - tar -cJf "$base.darwin.x86_64.tar.xz" --transform="s:^:$base/:" README.txt LICENSE.txt shellcheck + tar -cJf "$base.$os.$arch.tar.xz" --transform="s:^:$base/:" README.txt LICENSE.txt shellcheck rm "shellcheck" done From 65044c25681038dc061ee125d161b1325abbec10 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 28 Oct 2020 13:36:10 -0700 Subject: [PATCH 399/763] SC2095: Also warn if the command is backgrounded --- src/ShellCheck/Analytics.hs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index c894075..ed626d9 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2361,6 +2361,7 @@ prop_checkWhileReadPitfalls11 = verifyNot checkWhileReadPitfalls "while read foo prop_checkWhileReadPitfalls12 = verifyNot checkWhileReadPitfalls "while read foo\ndo\nmplayer foo.ogv << EOF\nq\nEOF\ndone" prop_checkWhileReadPitfalls13 = verify checkWhileReadPitfalls "while read foo; do x=$(ssh host cmd); done" prop_checkWhileReadPitfalls14 = verify checkWhileReadPitfalls "while read foo; do echo $(ssh host cmd) < /dev/null; done" +prop_checkWhileReadPitfalls15 = verify checkWhileReadPitfalls "while read foo; do ssh $foo cmd & done" checkWhileReadPitfalls params (T_WhileExpression id [command] contents) | isStdinReadCommand command = @@ -2411,6 +2412,7 @@ checkWhileReadPitfalls params (T_WhileExpression id [command] contents) warnWithFix (getId cmd) 2095 ("Use " ++ name ++ " " ++ flag ++ " to prevent " ++ name ++ " from swallowing stdin.") (fix flag cmd) + checkMuncher (T_Backgrounded _ t) = checkMuncher t checkMuncher _ = return () stdinRedirect (T_FdRedirect _ fd op) From 18e80284ec4f186e9cb061fb3a5210ed9901fb4c Mon Sep 17 00:00:00 2001 From: George Plymale II Date: Tue, 1 Dec 2020 16:15:22 -0500 Subject: [PATCH 400/763] add macports as installation option in README.md --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 05ce5ad..5459c53 100644 --- a/README.md +++ b/README.md @@ -172,6 +172,10 @@ On macOS (OS X) with Homebrew: brew install shellcheck +Or with MacPorts: + + sudo port install shellcheck + On OpenBSD: pkg_add shellcheck From b625562d609db2d5e10c0218445ada1f22e0b52f Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 2 Dec 2020 18:43:56 -0800 Subject: [PATCH 401/763] Add POSIX checks for more Bash-specific variables (fixes #2093) --- src/ShellCheck/Checks/ShellSupport.hs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index c4db9ce..1d373e5 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -417,9 +417,15 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do (re $ "^[" ++ varChars ++ "*@]+(\\[.*\\])?/", 3060, "string replacement is") ] bashVars = [ + -- This list deliberately excludes $BASH_VERSION as it's often used + -- for shell identification. "OSTYPE", "MACHTYPE", "HOSTTYPE", "HOSTNAME", "DIRSTACK", "EUID", "UID", "SHLVL", "PIPESTATUS", "SHELLOPTS", - "_" + "_", "BASHOPTS", "BASHPID", "BASH_ALIASES", "BASH_ARGC", + "BASH_ARGV", "BASH_ARGV0", "BASH_CMDS", "BASH_COMMAND", + "BASH_EXECUTION_STRING", "BASH_LINENO", "BASH_REMATCH", "BASH_SOURCE", + "BASH_SUBSHELL", "BASH_VERSINFO", "EPOCHREALTIME", "EPOCHSECONDS", + "FUNCNAME", "GROUPS", "MACHTYPE", "MAPFILE" ] bashDynamicVars = [ "RANDOM", "SECONDS" ] dashVars = [ "_" ] From 8e332ce87978e6f6afdfabcc15b382983ea32550 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 6 Dec 2020 20:30:43 -0800 Subject: [PATCH 402/763] Improve handling of trailing tokens for []/compounds (fixes #2091) --- CHANGELOG.md | 1 + src/ShellCheck/Parser.hs | 66 +++++++++++++++++++++++++++++----------- 2 files changed, 50 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e0eff4..fbedc20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ ### Fixed - SC1072/SC1073 now respond to disable annotations, though ignoring parse errors is still purely cosmetic and does not allow ShellCheck to continue. +- Improved error reporting for trailing tokens after ]/]] and compound commands ### Changed - Assignments are now parsed to spec, without leniency for leading $ or spaces diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 87efa25..dbadc7c 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -392,6 +392,8 @@ unexpecting s p = try $ notFollowedBy2 = unexpecting "" +isFollowedBy p = (lookAhead . try $ p *> return True) <|> return False + reluctantlyTill p end = (lookAhead (void (try end) <|> eof) >> return []) <|> do x <- p @@ -923,8 +925,9 @@ prop_readCondition20 = isOk readCondition "[[ echo_rc -eq 0 ]]" prop_readCondition21 = isOk readCondition "[[ $1 =~ ^(a\\ b)$ ]]" prop_readCondition22 = isOk readCondition "[[ $1 =~ \\.a\\.(\\.b\\.)\\.c\\. ]]" prop_readCondition23 = isOk readCondition "[[ -v arr[$var] ]]" -prop_readCondition24 = isWarning readCondition "[[ 1 == 2 ]]]" prop_readCondition25 = isOk readCondition "[[ lex.yy.c -ot program.l ]]" +prop_readCondition26 = isOk readScript "[[ foo ]]\\\n && bar" +prop_readCondition27 = not $ isOk readConditionCommand "[[ x ]] foo" readCondition = called "test expression" $ do opos <- getPosition start <- startSpan @@ -953,13 +956,7 @@ readCondition = called "test expression" $ do id <- endSpan start when (open == "[[" && close /= "]]") $ parseProblemAt cpos ErrorC 1033 "Test expression was opened with double [[ but closed with single ]. Make sure they match." when (open == "[" && close /= "]" ) $ parseProblemAt opos ErrorC 1034 "Test expression was opened with single [ but closed with double ]]. Make sure they match." - optional $ lookAhead $ do - pos <- getPosition - notFollowedBy2 readCmdWord <|> - parseProblemAt pos ErrorC 1136 - ("Unexpected characters after terminating " ++ close ++ ". Missing semicolon/linefeed?") spacing - many readCmdWord -- Read and throw away remainders to get then/do warnings. Fixme? return $ T_Condition id typ condition readAnnotationPrefix = do @@ -1617,6 +1614,7 @@ readArithmeticExpression = called "((..)) command" $ do c <- readArithmeticContents string "))" id <- endSpan start + spacing return (T_Arithmetic id c) -- If the next characters match prefix, try two different parsers and warn if the alternate parser had to be used @@ -1969,8 +1967,6 @@ readIoRedirect = do spacing return $ T_FdRedirect id n redir -readRedirectList = many1 readIoRedirect - prop_readHereString = isOk readHereString "<<< \"Hello $world\"" readHereString = called "here string" $ do start <- startSpan @@ -2300,6 +2296,7 @@ readPipe = do readCommand = choice [ readCompoundCommand, + readConditionCommand, readCoProc, readSimpleCommand ] @@ -2412,6 +2409,7 @@ readSubshell = called "explicit subshell" $ do allspacing char ')' <|> fail "Expected ) closing the subshell" id <- endSpan start + spacing return $ T_Subshell id list prop_readBraceGroup = isOk readBraceGroup "{ a; b | c | d; e; }" @@ -2432,6 +2430,7 @@ readBraceGroup = called "brace group" $ do parseProblem ErrorC 1056 "Expected a '}'. If you have one, try a ; or \\n in front of it." fail "Missing '}'" id <- endSpan start + spacing return $ T_BraceGroup id list prop_readBatsTest = isOk readBatsTest "@test 'can parse' {\n true\n}" @@ -2484,6 +2483,11 @@ readDoGroup kwId = do parseProblemAtId (getId doKw) ErrorC 1061 "Couldn't find 'done' for this 'do'." parseProblem ErrorC 1062 "Expected 'done' matching previously mentioned 'do'." return "Expected 'done'" + + optional . lookAhead $ do + pos <- getPosition + try $ string "<(" + parseProblemAt pos ErrorC 1142 "Use 'done < <(cmd)' to redirect from process substitution (currently missing one '<')." return commands @@ -2701,9 +2705,38 @@ readCoProc = called "coproc" $ do id <- endSpan start return $ T_CoProcBody id body - readPattern = (readPatternWord `thenSkip` spacing) `sepBy1` (char '|' `thenSkip` spacing) +prop_readConditionCommand = isOk readConditionCommand "[[ x ]] > foo 2>&1" +readConditionCommand = do + cmd <- readCondition + redirs <- many readIoRedirect + id <- getNextIdSpanningTokenList (cmd:redirs) + + pos <- getPosition + hasDashAo <- isFollowedBy $ do + c <- choice $ map (\s -> try $ string s) ["-o", "-a", "or", "and"] + posEnd <- getPosition + parseProblemAtWithEnd pos posEnd ErrorC 1139 $ + "Use " ++ alt c ++ " instead of '" ++ c ++ "' between test commands." + + -- If the next word is a keyword, readNormalWord will trigger a warning + hasKeyword <- isFollowedBy readKeyword + hasWord <- isFollowedBy readNormalWord + + when (hasWord && not (hasKeyword || hasDashAo)) $ do + -- We have other words following, and no error has been emitted. + posEnd <- getPosition + parseProblemAtWithEnd pos posEnd ErrorC 1140 "Unexpected parameters after condition. Missing &&/||, or bad expression?" + + return $ T_Redirecting id redirs cmd + where + alt "or" = "||" + alt "-o" = "||" + alt "and" = "&&" + alt "-a" = "&&" + alt _ = "|| or &&" + prop_readCompoundCommand = isOk readCompoundCommand "{ echo foo; }>/dev/null" readCompoundCommand = do cmd <- choice [ @@ -2711,7 +2744,6 @@ readCompoundCommand = do readAmbiguous "((" readArithmeticExpression readSubshell (\pos -> parseNoteAt pos ErrorC 1105 "Shells disambiguate (( differently or not at all. For subshell, add spaces around ( . For ((, fix parsing errors."), readSubshell, - readCondition, readWhileClause, readUntilClause, readIfClause, @@ -2721,15 +2753,15 @@ readCompoundCommand = do readBatsTest, readFunctionDefinition ] - spacing redirs <- many readIoRedirect id <- getNextIdSpanningTokenList (cmd:redirs) - unless (null redirs) $ optional $ do - lookAhead $ try (spacing >> needsSeparator) - parseProblem WarningC 1013 "Bash requires ; or \\n here, after redirecting nested compound commands." + optional . lookAhead $ do + notFollowedBy2 $ choice [readKeyword, g_Lbrace] + pos <- getPosition + many1 readNormalWord + posEnd <- getPosition + parseProblemAtWithEnd pos posEnd ErrorC 1141 "Unexpected tokens after compound command. Bad redirection or missing ;/&&/||/|?" return $ T_Redirecting id redirs cmd - where - needsSeparator = choice [ g_Then, g_Else, g_Elif, g_Fi, g_Do, g_Done, g_Esac, g_Rbrace ] readCompoundList = readTerm From 6ba1af089845a2c280a24e2dab58129391481175 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Fri, 11 Dec 2020 20:28:36 -0800 Subject: [PATCH 403/763] Warn when a variable is assigned to itself --- src/ShellCheck/Analytics.hs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index ed626d9..ef1ed2c 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -193,6 +193,7 @@ nodeChecks = [ ,checkModifiedArithmeticInRedirection ,checkBlatantRecursion ,checkBadTestAndOr + ,checkAssignToSelf ] optionalChecks = map fst optionalTreeChecks @@ -3974,5 +3975,26 @@ checkComparisonWithLeadingX params t = return $ replaceStart id params 2 "'" _ -> Nothing +prop_checkAssignToSelf1 = verify checkAssignToSelf "x=$x" +prop_checkAssignToSelf2 = verify checkAssignToSelf "x=${x}" +prop_checkAssignToSelf3 = verify checkAssignToSelf "x=\"$x\"" +prop_checkAssignToSelf4 = verifyNot checkAssignToSelf "x=$x mycmd" +checkAssignToSelf _ t = + case t of + T_SimpleCommand _ vars [] -> mapM_ check vars + _ -> return () + where + check t = + case t of + T_Assignment id Assign name [] t -> + case getWordParts t of + [T_DollarBraced _ _ b] -> do + when (Just name == getLiteralString b) $ + msg id + _ -> return () + _ -> return () + msg id = info id 2269 "This variable is assigned to itself, so the assignment does nothing." + + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) From cc3884cf9f81c7b709e89850167b59e03ac91266 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 12 Dec 2020 20:24:32 -0800 Subject: [PATCH 404/763] Support env -S/--split-string in shebangs (fixes #2105) --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 7 ++++++- src/ShellCheck/AnalyzerLib.hs | 8 ++++++-- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fbedc20..0617ab0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ - SC1072/SC1073 now respond to disable annotations, though ignoring parse errors is still purely cosmetic and does not allow ShellCheck to continue. - Improved error reporting for trailing tokens after ]/]] and compound commands +- `#!/usr/bin/env -S shell` is now handled correctly ### Changed - Assignments are now parsed to spec, without leniency for leading $ or spaces diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index ef1ed2c..2e1adca 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -546,9 +546,14 @@ indexOfSublists sub = f 0 prop_checkShebangParameters1 = verifyTree checkShebangParameters "#!/usr/bin/env bash -x\necho cow" prop_checkShebangParameters2 = verifyNotTree checkShebangParameters "#! /bin/sh -l " +prop_checkShebangParameters3 = verifyNotTree checkShebangParameters "#!/usr/bin/env -S bash -x\necho cow" +prop_checkShebangParameters4 = verifyNotTree checkShebangParameters "#!/usr/bin/env --split-string bash -x\necho cow" checkShebangParameters p (T_Annotation _ _ t) = checkShebangParameters p t checkShebangParameters _ (T_Script _ (T_Literal id sb) _) = - [makeComment ErrorC id 2096 "On most OS, shebangs can only specify a single parameter." | length (words sb) > 2] + [makeComment ErrorC id 2096 "On most OS, shebangs can only specify a single parameter." | isMultiWord] + where + isMultiWord = length (words sb) > 2 && not (sb `matches` re) + re = mkRegex "env +(-S|--split-string)" prop_checkShebang1 = verifyNotTree checkShebang "#!/usr/bin/env bash -x\necho cow" prop_checkShebang2 = verifyNotTree checkShebang "#! /bin/sh -l " diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index dc081db..38e9c1d 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -238,6 +238,8 @@ prop_determineShell5 = determineShellTest "#shellcheck shell=sh\nfoo" == Sh prop_determineShell6 = determineShellTest "#! /bin/sh" == Sh prop_determineShell7 = determineShellTest "#! /bin/ash" == Dash prop_determineShell8 = determineShellTest' (Just Ksh) "#!/bin/sh" == Sh +prop_determineShell9 = determineShellTest "#!/bin/env -S dash -x" == Dash +prop_determineShell10 = determineShellTest "#!/bin/env --split-string= dash -x" == Dash determineShellTest = determineShellTest' Nothing determineShellTest' fallbackShell = determineShell fallbackShell . fromJust . prRoot . pScript @@ -256,10 +258,12 @@ determineShell fallbackShell t = fromMaybe Bash $ executableFromShebang :: String -> String executableFromShebang = shellFor where - shellFor s | "/env " `isInfixOf` s = headOrDefault "" (drop 1 $ words s) + shellFor s | "/env " `isInfixOf` s = fromMaybe "" $ do + [flag, shell] <- matchRegex re s + return shell shellFor s | ' ' `elem` s = shellFor $ takeWhile (/= ' ') s shellFor s = reverse . takeWhile (/= '/') . reverse $ s - + re = mkRegex "/env +(-S|--split-string=?)? *([^ ]*)" -- Given a root node, make a map from Id to parent Token. From bd3299edd3b517f92f74c2e3327c9f6b72b31f7c Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 17 Dec 2020 20:30:39 -0800 Subject: [PATCH 405/763] Treat 'exec $1' like '$1' for the purpose of quoting (fixes #2068) --- src/ShellCheck/ASTLib.hs | 1 + src/ShellCheck/Analytics.hs | 2 ++ src/ShellCheck/AnalyzerLib.hs | 4 ++-- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index a09bf38..dcc9904 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -422,6 +422,7 @@ getCommandNameAndToken direct t = fromMaybe (Nothing, t) $ do "busybox" -> firstArg "builtin" -> firstArg "command" -> firstArg + "run" -> firstArg -- Used by bats "exec" -> do opts <- getBsdOpts "cla:" args (_, (t, _)) <- listToMaybe $ filter (null . fst) opts diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 2e1adca..2fb1253 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1853,6 +1853,8 @@ prop_checkSpacefulness37v = verifyTree checkVerboseSpacefulness "@test 'status' prop_checkSpacefulness38= verifyTree checkSpacefulness "a=; echo $a" prop_checkSpacefulness39= verifyNotTree checkSpacefulness "a=''\"\"''; b=x$a; echo $b" prop_checkSpacefulness40= verifyNotTree checkSpacefulness "a=$((x+1)); echo $a" +prop_checkSpacefulness41= verifyNotTree checkSpacefulness "exec $1 --flags" +prop_checkSpacefulness42= verifyNotTree checkSpacefulness "run $1 --flags" data SpaceStatus = SpaceSome | SpaceNone | SpaceEmpty deriving (Eq) instance Semigroup SpaceStatus where diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 38e9c1d..194bf18 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -370,8 +370,8 @@ usedAsCommandName tree token = go (getId token) (tail $ getPath tree token) | currentId == getId word = go id rest go currentId (T_DoubleQuoted id [word]:rest) | currentId == getId word = go id rest - go currentId (T_SimpleCommand _ _ (word:_):_) - | currentId == getId word = True + go currentId (t@(T_SimpleCommand _ _ (word:_)):_) = + getId word == currentId || getId (getCommandTokenOrThis t) == currentId go _ _ = False -- A list of the element and all its parents up to the root node. From 4e7e3f94565a5423aca0660ee822308d16a7f722 Mon Sep 17 00:00:00 2001 From: Pepe Iborra Date: Tue, 22 Dec 2020 09:15:57 +0000 Subject: [PATCH 406/763] Add Haddock markup to SystemInterface --- src/ShellCheck/Interface.hs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ShellCheck/Interface.hs b/src/ShellCheck/Interface.hs index 85d25c0..87346a1 100644 --- a/src/ShellCheck/Interface.hs +++ b/src/ShellCheck/Interface.hs @@ -73,15 +73,15 @@ import qualified Data.Map as Map data SystemInterface m = SystemInterface { - -- Read a file by filename, or return an error + -- | Read a file by filename, or return an error siReadFile :: String -> m (Either ErrorMessage String), - -- Given: + -- | Given: -- the current script, -- a list of source-path annotations in effect, -- and a sourced file, -- find the sourced file siFindSource :: String -> [String] -> String -> m FilePath, - -- Get the configuration file (name, contents) for a filename + -- | Get the configuration file (name, contents) for a filename siGetConfig :: String -> m (Maybe (FilePath, String)) } From 19355226e1d88abac37bb2eb5f366110ead0ecb1 Mon Sep 17 00:00:00 2001 From: Martin Bagge / brother Date: Sun, 27 Dec 2020 00:27:36 +0100 Subject: [PATCH 407/763] Change error 2076 to a warning. Implementing the suggestion by @pixarbuff #1985. --- src/ShellCheck/Analytics.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 2fb1253..a2ab577 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1193,8 +1193,8 @@ checkQuotedCondRegex _ (TC_Binary _ _ "=~" _ rhs) = where error t = unless (isConstantNonRe t) $ - err (getId t) 2076 - "Don't quote right-hand side of =~, it'll match literally rather than as a regex." + warn (getId t) 2076 + "Remove quotes from right-hand side of =~ to match as a regex rather than literally." re = mkRegex "[][*.+()|]" hasMetachars s = s `matches` re isConstantNonRe t = fromMaybe False $ do From 35033a9f2fdca88b916b62bbff284c218167d8f4 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Mon, 28 Dec 2020 16:22:53 -0500 Subject: [PATCH 408/763] Remove unnecessary use of Maybe from shellFor --- src/ShellCheck/AnalyzerLib.hs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 194bf18..9265a3f 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -258,9 +258,9 @@ determineShell fallbackShell t = fromMaybe Bash $ executableFromShebang :: String -> String executableFromShebang = shellFor where - shellFor s | "/env " `isInfixOf` s = fromMaybe "" $ do - [flag, shell] <- matchRegex re s - return shell + shellFor s | "/env " `isInfixOf` s = case matchRegex re s of + Just [flag, shell] -> shell + _ -> "" shellFor s | ' ' `elem` s = shellFor $ takeWhile (/= ' ') s shellFor s = reverse . takeWhile (/= '/') . reverse $ s re = mkRegex "/env +(-S|--split-string=?)? *([^ ]*)" From eaccd3d02c2e9c2f64d796801220ced38cce57fd Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Mon, 28 Dec 2020 16:32:10 -0500 Subject: [PATCH 409/763] Simplify parser --- src/ShellCheck/Parser.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index dbadc7c..104e0a4 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -392,7 +392,7 @@ unexpecting s p = try $ notFollowedBy2 = unexpecting "" -isFollowedBy p = (lookAhead . try $ p *> return True) <|> return False +isFollowedBy p = (lookAhead . try $ p $> True) <|> return False reluctantlyTill p end = (lookAhead (void (try end) <|> eof) >> return []) <|> do @@ -2715,7 +2715,7 @@ readConditionCommand = do pos <- getPosition hasDashAo <- isFollowedBy $ do - c <- choice $ map (\s -> try $ string s) ["-o", "-a", "or", "and"] + c <- choice $ try . string <$> ["-o", "-a", "or", "and"] posEnd <- getPosition parseProblemAtWithEnd pos posEnd ErrorC 1139 $ "Use " ++ alt c ++ " instead of '" ++ c ++ "' between test commands." From 46f177b5beb5c9862bfd5e3d10cc25a844af4c4c Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Mon, 28 Dec 2020 17:14:18 -0500 Subject: [PATCH 410/763] Simplify parseArgs --- src/ShellCheck/AnalyzerLib.hs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 9265a3f..a1e089a 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -687,12 +687,10 @@ getModifiedVariableCommand base@(T_SimpleCommand id cmdPrefix (T_NormalWord _ (T parseArgs :: Maybe (Token, Token, String, DataType) parseArgs = do args <- getGnuOpts "d:n:O:s:u:C:c:t" rest - let names = map snd $ filter (\(x,y) -> null x) args - if null names - then + case [y | ("",(_,y)) <- args] of + [] -> return (base, base, "MAPFILE", DataArray SourceExternal) - else do - (_, first) <- listToMaybe names + first:_ -> do name <- getLiteralString first guard $ isVariableName name return (base, first, name, DataArray SourceExternal) From 0607039d419a075df594fd5c38153e2d4863e834 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Mon, 28 Dec 2020 17:21:47 -0500 Subject: [PATCH 411/763] Simplify actualArgs --- src/ShellCheck/Checks/Commands.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index aa4edd4..f1996ef 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -122,7 +122,7 @@ checkGetOpts str flags args f = toTokens = map (T_Literal (Id 0)) . words opts = fromMaybe [] $ f (toTokens str) actualFlags = filter (not . null) $ map fst opts - actualArgs = map (\(_, (_, x)) -> onlyLiteralString x) $ filter (null . fst) opts + actualArgs = [onlyLiteralString x | ("", (_, x)) <- opts] -- Short options prop_checkGetOptsS1 = checkGetOpts "-f x" ["f"] [] $ getOpts (True, True) "f:" [] From cb4f4e7edc8721e12bd7a4ace9f88e8468961c53 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Mon, 28 Dec 2020 17:34:52 -0500 Subject: [PATCH 412/763] Use mapM_ instead of reimplementing it --- src/ShellCheck/Analytics.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 2fb1253..3d5692a 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -3808,7 +3808,7 @@ checkAliasUsedInSameParsingUnit params root = -- Group them by whether they start on the same line where the previous one ended units = groupByLink followsOnLine commands in - execWriter $ sequence_ $ map checkUnit units + execWriter $ mapM_ checkUnit units where lineSpan t = let m = tokenPositions params in do From 2c0766825e3e25589467c03435416f2fa3d9062f Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Mon, 28 Dec 2020 17:45:11 -0500 Subject: [PATCH 413/763] Implement groupByLink in terms of foldr --- src/ShellCheck/Analytics.hs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 3d5692a..70d7f8e 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -3858,13 +3858,13 @@ groupByLink :: (a -> a -> Bool) -> [a] -> [[a]] groupByLink f list = case list of [] -> [] - (x:xs) -> g x [] xs + (x:xs) -> foldr c n xs x [] where - g current span (next:rest) = + c next rest current span = if f current next - then g next (current:span) rest - else (reverse $ current:span) : g next [] rest - g current span [] = [reverse (current:span)] + then rest next (current:span) + else (reverse $ current:span) : rest next [] + n current span = [reverse (current:span)] prop_checkBlatantRecursion1 = verify checkBlatantRecursion ":(){ :|:& };:" From dfbcc9595e0b80e25c61bfedb7ad0347ff045395 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Mon, 28 Dec 2020 17:48:58 -0500 Subject: [PATCH 414/763] Use mapM instead of reimplementing it --- src/ShellCheck/Analytics.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 70d7f8e..945397c 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -3371,7 +3371,7 @@ checkPipeToNowhere params t = sequence_ $ do T_Redirecting _ redirs cmd <- return stage - fds <- sequence $ map getRedirectionFds redirs + fds <- mapM getRedirectionFds redirs let fdAndToken :: [(Integer, Token)] fdAndToken = From 848056367267e75cb3db0e45ec282a58105c0ed0 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Mon, 28 Dec 2020 17:55:54 -0500 Subject: [PATCH 415/763] Use syntactic sugar instead of building lists by hand --- src/ShellCheck/ASTLib.hs | 2 +- src/ShellCheck/Analytics.hs | 6 +++--- src/ShellCheck/Checks/Commands.hs | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index dcc9904..6e48bf3 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -177,7 +177,7 @@ getOpts (gnu, arbitraryLongOpts) string longopts args = process args process [] = return [] process (token:rest) = do case getLiteralStringDef "\0" token of - '-':'-':[] -> return $ listToArgs rest + "--" -> return $ listToArgs rest '-':'-':word -> do let (name, arg) = span (/= '=') word needsArg <- diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 945397c..c8473eb 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -394,7 +394,7 @@ prop_checkAssignAteCommand4 = verifyNot checkAssignAteCommand "A=foo ls -l" prop_checkAssignAteCommand5 = verify checkAssignAteCommand "PAGER=cat grep bar" prop_checkAssignAteCommand6 = verifyNot checkAssignAteCommand "PAGER=\"cat\" grep bar" prop_checkAssignAteCommand7 = verify checkAssignAteCommand "here=pwd" -checkAssignAteCommand _ (T_SimpleCommand id (T_Assignment _ _ _ _ assignmentTerm:[]) list) = +checkAssignAteCommand _ (T_SimpleCommand id [T_Assignment _ _ _ _ assignmentTerm] list) = -- Check if first word is intended as an argument (flag or glob). if firstWordIsArg list then @@ -426,7 +426,7 @@ checkArithmeticOpCommand _ _ = return () prop_checkWrongArit = verify checkWrongArithmeticAssignment "i=i+1" prop_checkWrongArit2 = verify checkWrongArithmeticAssignment "n=2; i=n*2" -checkWrongArithmeticAssignment params (T_SimpleCommand id (T_Assignment _ _ _ _ val:[]) []) = +checkWrongArithmeticAssignment params (T_SimpleCommand id [T_Assignment _ _ _ _ val] []) = sequence_ $ do str <- getNormalString val match <- matchRegex regex str @@ -2844,7 +2844,7 @@ checkTestArgumentSplitting params t = then -- Ksh appears to stop processing after unrecognized tokens, so operators -- will effectively work with globs, but only the first match. - when (op `elem` ['-':c:[] | c <- "bcdfgkprsuwxLhNOGRS" ]) $ + when (op `elem` [['-', c] | c <- "bcdfgkprsuwxLhNOGRS" ]) $ warn (getId token) 2245 $ op ++ " only applies to the first expansion of this glob. Use a loop to check any/all." else diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index f1996ef..771083e 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -916,7 +916,7 @@ checkWhileGetoptsCase = CommandCheck (Exactly "getopts") f fromGlob t = case t of - T_Glob _ ('[':c:']':[]) -> return [c] + T_Glob _ ['[', c, ']'] -> return [c] T_Glob _ "*" -> return "*" T_Glob _ "?" -> return "?" _ -> Nothing From e7820479f0502ea4571ec797d286c9037a81d967 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Mon, 28 Dec 2020 17:56:25 -0500 Subject: [PATCH 416/763] Use find --- src/ShellCheck/ASTLib.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 6e48bf3..c038ebe 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -425,7 +425,7 @@ getCommandNameAndToken direct t = fromMaybe (Nothing, t) $ do "run" -> firstArg -- Used by bats "exec" -> do opts <- getBsdOpts "cla:" args - (_, (t, _)) <- listToMaybe $ filter (null . fst) opts + (_, (t, _)) <- find (null . fst) opts return t _ -> fail "" From 34939ca0b7dce27472543466eeab1f68df317274 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Mon, 28 Dec 2020 18:00:14 -0500 Subject: [PATCH 417/763] Fuse map into any --- src/ShellCheck/Analytics.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index c8473eb..98fd5d7 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2481,7 +2481,7 @@ checkCharRangeGlob p t@(T_Glob id str) | where isCharClass str = "[" `isPrefixOf` str && "]" `isSuffixOf` str contents = dropNegation . drop 1 . take (length str - 1) $ str - hasDupes = any (>1) . map length . group . sort . filter (/= '-') $ contents + hasDupes = any ((>1) . length) . group . sort . filter (/= '-') $ contents dropNegation s = case s of '!':rest -> rest @@ -3404,7 +3404,7 @@ checkPipeToNowhere params t = commandSpecificException name cmd = case name of - "du" -> any (`elem` ["exclude-from", "files0-from"]) $ map snd $ getAllFlags cmd + "du" -> any ((`elem` ["exclude-from", "files0-from"]) . snd) $ getAllFlags cmd _ -> False warnAboutDupes (n, list@(_:_:_)) = From 81e84c293944292dfb05cf09048a5eb8bbfca40b Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Mon, 28 Dec 2020 18:03:14 -0500 Subject: [PATCH 418/763] Use execState instead of snd . runState --- src/ShellCheck/AnalyzerLib.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index a1e089a..f8e490e 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -270,7 +270,7 @@ executableFromShebang = shellFor -- This is used to populate parentMap in Parameters getParentTree :: Token -> Map.Map Id Token getParentTree t = - snd . snd $ runState (doStackAnalysis pre post t) ([], Map.empty) + snd $ execState (doStackAnalysis pre post t) ([], Map.empty) where pre t = modify (first ((:) t)) post t = do From e272fa04eed3e3f205e048a207d99e4cb011efcd Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Mon, 28 Dec 2020 18:04:32 -0500 Subject: [PATCH 419/763] Remove redundant bind and return --- src/ShellCheck/Parser.hs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 104e0a4..a17be91 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -209,8 +209,7 @@ startSpan = IncompleteInterval <$> getPosition endSpan (IncompleteInterval start) = do endPos <- getPosition - id <- getNextIdBetween start endPos - return id + getNextIdBetween start endPos getSpanPositionsFor m = do start <- getPosition From 953d9bc56dedba7a09829f2442204ae4b23074e1 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Mon, 28 Dec 2020 18:05:55 -0500 Subject: [PATCH 420/763] Remove unused helper stub --- src/ShellCheck/Parser.hs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index a17be91..48d1cfa 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -2088,10 +2088,6 @@ readSimpleCommand = called "simple command" $ do then action else getParser def cmd rest - cStyleComment cmd = - case cmd of - _ -> False - validateCommand cmd = case cmd of (T_NormalWord _ [T_Literal _ "//"]) -> commentWarning (getId cmd) From 2cfd1f27140d15132a031ef71a715e522a40837b Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Mon, 28 Dec 2020 18:10:47 -0500 Subject: [PATCH 421/763] Fuse maps --- shellcheck.hs | 2 +- src/ShellCheck/Checks/Commands.hs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/shellcheck.hs b/shellcheck.hs index f1757ef..d7e818d 100644 --- a/shellcheck.hs +++ b/shellcheck.hs @@ -507,7 +507,7 @@ ioInterface options files = do where find filename deflt = do sources <- findM ((allowable inputs) `andM` doesFileExist) $ - (adjustPath filename):(map ( filename) $ map adjustPath $ sourcePathFlag ++ sourcePathAnnotation) + (adjustPath filename):(map (( filename) . adjustPath) $ sourcePathFlag ++ sourcePathAnnotation) case sources of Nothing -> return deflt Just first -> return first diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 771083e..5be5a9a 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -951,7 +951,7 @@ checkCatastrophicRm = CommandCheck (Basename "rm") $ \t -> when (isRecursive t) $ mapM_ (mapM_ checkWord . braceExpand) $ arguments t where - isRecursive = any (`elem` ["r", "R", "recursive"]) . map snd . getAllFlags + isRecursive = any ((`elem` ["r", "R", "recursive"]) . snd) . getAllFlags checkWord token = case getLiteralString token of From fbb14d6b384a65ee16781966d9d240db9ed7b644 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 30 Dec 2020 19:24:14 -0800 Subject: [PATCH 422/763] Improve checks for = in command names (fixes #2102) --- CHANGELOG.md | 1 + src/ShellCheck/ASTLib.hs | 11 +- src/ShellCheck/Analytics.hs | 258 +++++++++++++++++++++++++++++++++++- src/ShellCheck/Parser.hs | 26 ++-- 4 files changed, 270 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0617ab0..596db82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ - POSIX/dash unsupported feature warnings now have individual SC3xxx codes - SC1090: A leading `$x/` or `$(x)/` is now treated as `./` when locating files - SC2154: Variables appearing in -z/-n tests are no longer considered unassigned +- SC2270-SC2285: Improved warnings about misused =, e.g. `${var}=42` ## v0.7.1 - 2020-04-04 diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index dcc9904..67d09f4 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -278,6 +278,12 @@ getUnquotedLiteral (T_NormalWord _ list) = str _ = Nothing getUnquotedLiteral _ = Nothing +isQuotes t = + case t of + T_DoubleQuoted {} -> True + T_SingleQuoted {} -> True + _ -> False + -- Get the last unquoted T_Literal in a word like "${var}foo"THIS -- or nothing if the word does not end in an unquoted literal. getTrailingUnquotedLiteral :: Token -> Maybe Token @@ -296,8 +302,11 @@ getTrailingUnquotedLiteral t = getLeadingUnquotedString :: Token -> Maybe String getLeadingUnquotedString t = case t of - T_NormalWord _ ((T_Literal _ s) : _) -> return s + T_NormalWord _ ((T_Literal _ s) : rest) -> return $ s ++ from rest _ -> Nothing + where + from ((T_Literal _ s):rest) = s ++ from rest + from _ = "" -- Maybe get the literal string of this token and any globs in it. getGlobOrLiteralString = getLiteralStringExt f diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 2fb1253..b7a444d 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -194,6 +194,8 @@ nodeChecks = [ ,checkBlatantRecursion ,checkBadTestAndOr ,checkAssignToSelf + ,checkEqualsInCommand + ,checkSecondArgIsComparison ] optionalChecks = map fst optionalTreeChecks @@ -277,15 +279,23 @@ checkUnqualifiedCommand str f t@(T_SimpleCommand id _ (cmd:rest)) | t `isUnqualifiedCommand` str = f cmd rest checkUnqualifiedCommand _ _ _ = return () +verifyCodes :: (Parameters -> Token -> Writer [TokenComment] ()) -> [Code] -> String -> Bool +verifyCodes f l s = codes == Just l + where + treeCheck = runNodeAnalysis f + comments = runAndGetComments treeCheck s + codes = map (cCode . tcComment) <$> comments checkNode f = producesComments (runNodeAnalysis f) producesComments :: (Parameters -> Token -> [TokenComment]) -> String -> Maybe Bool -producesComments f s = do +producesComments f s = not . null <$> runAndGetComments f s + +runAndGetComments f s = do let pr = pScript s prRoot pr let spec = defaultSpec pr let params = makeParameters spec - return . not . null $ + return $ filterByAnnotation spec params $ runList spec [f] @@ -364,6 +374,19 @@ replaceEnd id params n r = repPrecedence = depth, repInsertionPoint = InsertBefore } +replaceToken id params r = + let tp = tokenPositions params + (start, end) = tp Map.! id + depth = length $ getPath (parentMap params) (T_EOF id) + in + newReplacement { + repStartPos = start, + repEndPos = end, + repString = r, + repPrecedence = depth, + repInsertionPoint = InsertBefore + } + surroundWidth id params s = fixWith [replaceStart id params 0 s, replaceEnd id params 0 s] fixWith fixes = newFix { fixReplacements = fixes } @@ -1855,6 +1878,7 @@ prop_checkSpacefulness39= verifyNotTree checkSpacefulness "a=''\"\"''; b=x$a; ec prop_checkSpacefulness40= verifyNotTree checkSpacefulness "a=$((x+1)); echo $a" prop_checkSpacefulness41= verifyNotTree checkSpacefulness "exec $1 --flags" prop_checkSpacefulness42= verifyNotTree checkSpacefulness "run $1 --flags" +prop_checkSpacefulness43= verifyNotTree checkSpacefulness "$foo=42" data SpaceStatus = SpaceSome | SpaceNone | SpaceEmpty deriving (Eq) instance Semigroup SpaceStatus where @@ -1879,9 +1903,10 @@ checkSpacefulness params = checkSpacefulness' onFind params emit $ makeComment InfoC (getId token) 2223 "This default assignment may cause DoS due to globbing. Quote it." else - emit $ makeCommentWithFix InfoC (getId token) 2086 - "Double quote to prevent globbing and word splitting." - (addDoubleQuotesAround params token) + unless (quotesMayConflictWithSC2281 params token) $ + emit $ makeCommentWithFix InfoC (getId token) 2086 + "Double quote to prevent globbing and word splitting." + (addDoubleQuotesAround params token) isDefaultAssignment parents token = let modifier = getBracedModifier $ bracedString token in @@ -1896,14 +1921,25 @@ prop_checkSpacefulness4v= verifyTree checkVerboseSpacefulness "foo=3; foo=$(echo prop_checkSpacefulness8v= verifyTree checkVerboseSpacefulness "a=foo\\ bar; a=foo; rm $a" prop_checkSpacefulness28v = verifyTree checkVerboseSpacefulness "exec {n}>&1; echo $n" prop_checkSpacefulness36v = verifyTree checkVerboseSpacefulness "arg=$#; echo $arg" +prop_checkSpacefulness44v = verifyNotTree checkVerboseSpacefulness "foo=3; $foo=4" checkVerboseSpacefulness params = checkSpacefulness' onFind params where onFind spaces token name = - when (spaces == SpaceNone && name `notElem` specialVariablesWithoutSpaces) $ + when (spaces == SpaceNone + && name `notElem` specialVariablesWithoutSpaces + && not (quotesMayConflictWithSC2281 params token)) $ tell [makeCommentWithFix StyleC (getId token) 2248 "Prefer double quoting even when variables don't contain special characters." (addDoubleQuotesAround params token)] +-- Don't suggest quotes if this will instead be autocorrected +-- from $foo=bar to foo=bar. This is not pretty but ok. +quotesMayConflictWithSC2281 params t = + case getPath (parentMap params) t of + _ : T_NormalWord parentId (me:T_Literal _ ('=':_):_) : T_SimpleCommand _ _ (cmd:_) : _ -> + (getId t) == (getId me) && (parentId == getId cmd) + _ -> False + addDoubleQuotesAround params token = (surroundWidth (getId token) params "\"") checkSpacefulness' :: (SpaceStatus -> Token -> String -> Writer [TokenComment] ()) -> @@ -1978,8 +2014,9 @@ prop_CheckVariableBraces1 = verify checkVariableBraces "a='123'; echo $a" prop_CheckVariableBraces2 = verifyNot checkVariableBraces "a='123'; echo ${a}" prop_CheckVariableBraces3 = verifyNot checkVariableBraces "#shellcheck disable=SC2016\necho '$a'" prop_CheckVariableBraces4 = verifyNot checkVariableBraces "echo $* $1" +prop_CheckVariableBraces5 = verifyNot checkVariableBraces "$foo=42" checkVariableBraces params t@(T_DollarBraced id False l) - | name `notElem` unbracedVariables = + | name `notElem` unbracedVariables && not (quotesMayConflictWithSC2281 params t) = styleWithFix id 2250 "Prefer putting braces around variable references even when not strictly required." (fixFor t) @@ -4003,5 +4040,212 @@ checkAssignToSelf _ t = msg id = info id 2269 "This variable is assigned to itself, so the assignment does nothing." +prop_checkEqualsInCommand1a = verifyCodes checkEqualsInCommand [2277] "#!/bin/bash\n0='foo'" +prop_checkEqualsInCommand2a = verifyCodes checkEqualsInCommand [2278] "#!/bin/ksh \n$0='foo'" +prop_checkEqualsInCommand3a = verifyCodes checkEqualsInCommand [2279] "#!/bin/dash\n${0}='foo'" +prop_checkEqualsInCommand4a = verifyCodes checkEqualsInCommand [2280] "#!/bin/sh \n0='foo'" + +prop_checkEqualsInCommand1b = verifyCodes checkEqualsInCommand [2270] "1='foo'" +prop_checkEqualsInCommand2b = verifyCodes checkEqualsInCommand [2270] "${2}='foo'" + +prop_checkEqualsInCommand1c = verifyCodes checkEqualsInCommand [2271] "var$((n+1))=value" +prop_checkEqualsInCommand2c = verifyCodes checkEqualsInCommand [2271] "var${x}=value" +prop_checkEqualsInCommand3c = verifyCodes checkEqualsInCommand [2271] "var$((cmd))x='foo'" +prop_checkEqualsInCommand4c = verifyCodes checkEqualsInCommand [2271] "$(cmd)='foo'" + +prop_checkEqualsInCommand1d = verifyCodes checkEqualsInCommand [2273] "=======" +prop_checkEqualsInCommand2d = verifyCodes checkEqualsInCommand [2274] "======= Here =======" +prop_checkEqualsInCommand3d = verifyCodes checkEqualsInCommand [2275] "foo\n=42" + +prop_checkEqualsInCommand1e = verifyCodes checkEqualsInCommand [] "--foo=bar" +prop_checkEqualsInCommand2e = verifyCodes checkEqualsInCommand [] "$(cmd)'=foo'" +prop_checkEqualsInCommand3e = verifyCodes checkEqualsInCommand [2276] "var${x}/=value" +prop_checkEqualsInCommand4e = verifyCodes checkEqualsInCommand [2276] "${}=value" +prop_checkEqualsInCommand5e = verifyCodes checkEqualsInCommand [2276] "${#x}=value" + +prop_checkEqualsInCommand1f = verifyCodes checkEqualsInCommand [2281] "$var=foo" +prop_checkEqualsInCommand2f = verifyCodes checkEqualsInCommand [2281] "$a=$b" +prop_checkEqualsInCommand3f = verifyCodes checkEqualsInCommand [2281] "${var}=foo" +prop_checkEqualsInCommand4f = verifyCodes checkEqualsInCommand [2281] "${var[42]}=foo" +prop_checkEqualsInCommand5f = verifyCodes checkEqualsInCommand [2281] "$var+=foo" + +prop_checkEqualsInCommand1g = verifyCodes checkEqualsInCommand [2282] "411toppm=true" + +checkEqualsInCommand params originalToken = + case originalToken of + T_SimpleCommand _ _ (word:_) -> check word + _ -> return () + where + hasEquals t = + case t of + T_Literal _ s -> '=' `elem` s + _ -> False + + check t@(T_NormalWord _ list) | any hasEquals list = + case break hasEquals list of + (leading, (eq:_)) -> msg t (stripSinglePlus leading) eq + _ -> return () + check _ = return () + + -- This is a workaround for the parser adding + and = as separate literals + stripSinglePlus l = + case reverse l of + (T_Literal _ "+"):rest -> reverse rest + _ -> l + + positionalAssignmentRe = mkRegex "^[0-9][0-9]?=" + positionalMsg id = + err id 2270 "To assign positional parameters, use 'set -- first second ..' (or use [ ] to compare)." + indirectionMsg id = + err id 2271 "For indirection, use arrays, declare \"var$n=value\", or (for sh) read/eval." + badComparisonMsg id = + err id 2272 "Command name contains ==. For comparison, use [ \"$var\" = value ]." + conflictMarkerMsg id = + err id 2273 "Sequence of ===s found. Merge conflict or intended as a commented border?" + borderMsg id = + err id 2274 "Command name starts with ===. Intended as a commented border?" + prefixMsg id = + err id 2275 "Command name starts with =. Bad line break?" + genericMsg id = + err id 2276 "This is interpreted as a command name containing '='. Bad assignment or comparison?" + assign0Msg id bashfix = + case shellType params of + Bash -> errWithFix id 2277 "Use BASH_ARGV0 to assign to $0 in bash (or use [ ] to compare)." bashfix + Ksh -> err id 2278 "$0 can't be assigned in Ksh (but it does reflect the current function)." + Dash -> err id 2279 "$0 can't be assigned in Dash. This becomes a command name." + _ -> err id 2280 "$0 can't be assigned this way, and there is no portable alternative." + leadingNumberMsg id = + err id 2282 "Variable names can't start with numbers, so this is interpreted as a command." + + isExpansion t = + case t of + T_Arithmetic {} -> True + _ -> isQuoteableExpansion t + + isConflictMarker cmd = fromMaybe False $ do + str <- getUnquotedLiteral cmd + guard $ all (== '=') str + guard $ length str >= 4 && length str <= 12 -- Git uses 7 but who knows + return True + + mayBeVariableName l = fromMaybe False $ do + guard . not $ any isQuotes l + guard . not $ any willBecomeMultipleArgs l + str <- getLiteralStringExt (\_ -> Just "x") (T_NormalWord (Id 0) l) + return $ isVariableName str + + isLeadingNumberVar s = + let lead = takeWhile (/= '=') s + in not (null lead) && isDigit (head lead) + && all isVariableChar lead && not (all isDigit lead) + + msg cmd leading (T_Literal litId s) = do + -- There are many different cases, and the order of the branches matter. + case leading of + -- --foo=42 + [] | "-" `isPrefixOf` s -> -- There's SC2215 for these + return () + + -- ======Hello====== + [] | "=" `isPrefixOf` s -> + case originalToken of + T_SimpleCommand _ [] [word] | isConflictMarker word -> + conflictMarkerMsg (getId originalToken) + _ | "===" `isPrefixOf` s -> borderMsg (getId originalToken) + _ -> prefixMsg (getId cmd) + + -- $var==42 + _ | "==" `isInfixOf` s -> + badComparisonMsg (getId cmd) + + -- ${foo[x]}=42 and $foo=42 + [T_DollarBraced id braced l] | "=" `isPrefixOf` s -> do + let variableStr = concat $ oversimplify l + let variableReference = getBracedReference variableStr + let variableModifier = getBracedModifier variableStr + let isPlain = isVariableName variableStr + let isPositional = all isDigit variableStr + + let isArray = variableReference /= "" + && "[" `isPrefixOf` variableModifier + && "]" `isSuffixOf` variableModifier + + case () of + -- $foo=bar should already have caused a parse-time SC1066 + -- _ | not braced && isPlain -> + -- return () + + _ | variableStr == "" -> -- Don't try to fix ${}=foo + genericMsg (getId cmd) + + -- $#=42 or ${#var}=42 + _ | "#" `isPrefixOf` variableStr -> + genericMsg (getId cmd) + + -- ${0}=42 + _ | variableStr == "0" -> + assign0Msg id $ fixWith [replaceToken id params "BASH_ARGV0"] + + -- $2=2 + _ | isPositional -> + positionalMsg id + + _ | isArray || isPlain -> + errWithFix id 2281 + ("Don't use " ++ (if braced then "${}" else "$") ++ " on the left side of assignments.") $ + fixWith $ + if braced + then [ replaceStart id params 2 "", replaceEnd id params 1 "" ] + else [ replaceStart id params 1 "" ] + + _ -> indirectionMsg id + + -- 2=42 + [] | s `matches` positionalAssignmentRe -> + if "0=" `isPrefixOf` s + then + assign0Msg litId $ fixWith [replaceStart litId params 1 "BASH_ARGV0"] + else + positionalMsg litId + + -- 9foo=42 + [] | isLeadingNumberVar s -> + leadingNumberMsg (getId cmd) + + -- var${foo}x=42 + (_:_) | mayBeVariableName leading && (all isVariableChar $ takeWhile (/= '=') s) -> + indirectionMsg (getId cmd) + + _ -> genericMsg (getId cmd) + + +prop_checkSecondArgIsComparison1 = verify checkSecondArgIsComparison "foo = $bar" +prop_checkSecondArgIsComparison2 = verify checkSecondArgIsComparison "$foo = $bar" +prop_checkSecondArgIsComparison3 = verify checkSecondArgIsComparison "2f == $bar" +prop_checkSecondArgIsComparison4 = verify checkSecondArgIsComparison "'var' =$bar" +prop_checkSecondArgIsComparison5 = verify checkSecondArgIsComparison "foo ='$bar'" +prop_checkSecondArgIsComparison6 = verify checkSecondArgIsComparison "$foo =$bar" +prop_checkSecondArgIsComparison7 = verify checkSecondArgIsComparison "2f ==$bar" +prop_checkSecondArgIsComparison8 = verify checkSecondArgIsComparison "'var' =$bar" +prop_checkSecondArgIsComparison9 = verify checkSecondArgIsComparison "var += $(foo)" +prop_checkSecondArgIsComparison10 = verify checkSecondArgIsComparison "var +=$(foo)" +checkSecondArgIsComparison _ t = + case t of + T_SimpleCommand _ _ (lhs:arg:_) -> sequence_ $ do + argString <- getLeadingUnquotedString arg + case argString of + '=':'=':'=':'=':_ -> Nothing -- Don't warn about `echo ======` and such + '+':'=':_ -> + return $ err (getId t) 2285 $ + "Remove spaces around += to assign (or quote '+=' if literal)." + '=':'=':_ -> + return $ err (getId t) 2284 $ + "Use [ x = y ] to compare values (or quote '==' if literal)." + '=':_ -> + return $ err (getId t) 2283 $ + "Use [ ] to compare values, or remove spaces around = to assign (or quote '=' if literal)." + _ -> Nothing + _ -> return () + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index dbadc7c..5634d95 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -2850,15 +2850,13 @@ readAssignmentWordExt lenient = called "variable assignment" $ do (id, variable, op, indices) <- try $ do start <- startSpan pos <- getPosition + -- Check for a leading $ at parse time, to warn for $foo=(bar) which + -- would otherwise cause a parse failure so it can't be checked later. leadingDollarPos <- if lenient then optionMaybe $ getSpanPositionsFor (char '$') else return Nothing variable <- readVariableName - middleDollarPos <- - if lenient - then optionMaybe $ getSpanPositionsFor readNormalDollar - else return Nothing indices <- many readArrayIndex hasLeftSpace <- fmap (not . null) spacing opStart <- getPosition @@ -2866,20 +2864,12 @@ readAssignmentWordExt lenient = called "variable assignment" $ do op <- readAssignmentOp opEnd <- getPosition - when (isJust leadingDollarPos || isJust middleDollarPos || hasLeftSpace) $ do - sequence_ $ do - (l, r) <- leadingDollarPos - return $ parseProblemAtWithEnd l r ErrorC 1066 "Don't use $ on the left side of assignments." - sequence_ $ do - (l, r) <- middleDollarPos - return $ parseProblemAtWithEnd l r ErrorC 1067 "For indirection, use arrays, declare \"var$n=value\", or (for sh) read/eval." - when hasLeftSpace $ do - parseProblemAtWithEnd opStart opEnd ErrorC 1068 $ - "Don't put spaces around the " - ++ (if op == Append - then "+= when appending" - else "= in assignments") - ++ " (or quote to make it literal)." + when (isJust leadingDollarPos || hasLeftSpace) $ do + hasParen <- isFollowedBy (spacing >> char '(') + when hasParen $ + sequence_ $ do + (l, r) <- leadingDollarPos + return $ parseProblemAtWithEnd l r ErrorC 1066 "Don't use $ on the left side of assignments." -- Fail so that this is not parsed as an assignment. fail "" From 5fbaae2bb3f9f26912e55d41894f592be82b4ec1 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 30 Dec 2020 20:55:18 -0800 Subject: [PATCH 423/763] Don't treat ${!x@} as reference of x (fixes #2116) --- src/ShellCheck/AnalyzerLib.hs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 194bf18..bc8bc3d 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -846,6 +846,7 @@ prop_getBracedReference8 = getBracedReference "foo-bar" == "foo" prop_getBracedReference9 = getBracedReference "foo:-bar" == "foo" prop_getBracedReference10= getBracedReference "foo: -1" == "foo" prop_getBracedReference11= getBracedReference "!os*" == "" +prop_getBracedReference11b= getBracedReference "!os@" == "" prop_getBracedReference12= getBracedReference "!os?bar**" == "" prop_getBracedReference13= getBracedReference "foo[bar]" == "foo" getBracedReference s = fromMaybe s $ @@ -864,7 +865,7 @@ getBracedReference s = fromMaybe s $ nameExpansion ('!':next:rest) = do -- e.g. ${!foo*bar*} guard $ isVariableChar next -- e.g. ${!@} first <- find (not . isVariableChar) rest - guard $ first `elem` "*?" + guard $ first `elem` "*?@" return "" nameExpansion _ = Nothing From 9584266a8b455c6d6d9cadc45a8647b1fef8ad74 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 31 Dec 2020 12:28:48 -0800 Subject: [PATCH 424/763] Escape control characters when adding user data to messages --- src/ShellCheck/ASTLib.hs | 32 +++++++++++++++++++++++++++++++ src/ShellCheck/Analytics.hs | 4 ++-- src/ShellCheck/Checks/Commands.hs | 2 +- src/ShellCheck/Parser.hs | 10 +++++----- 4 files changed, 40 insertions(+), 8 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 67d09f4..d9ee8b1 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -29,6 +29,7 @@ import Data.Functor.Identity import Data.List import Data.Maybe import qualified Data.Map as Map +import Numeric (showHex) arguments (T_SimpleCommand _ _ (cmd:args)) = args @@ -367,6 +368,37 @@ getLiteralStringExt more = g -- Is this token a string literal? isLiteral t = isJust $ getLiteralString t +-- Escape user data for messages. +-- Messages generally avoid repeating user data, but sometimes it's helpful. +e4m = escapeForMessage +escapeForMessage :: String -> String +escapeForMessage str = concatMap f str + where + f '\\' = "\\\\" + f '\n' = "\\n" + f '\r' = "\\r" + f '\t' = "\\t" + f '\x1B' = "\\e" + f c = + if shouldEscape c + then + if ord c < 256 + then "\\x" ++ (pad0 2 $ toHex c) + else "\\U" ++ (pad0 4 $ toHex c) + else [c] + + shouldEscape c = + (not $ isPrint c) + || (not (isAscii c) && not (isLetter c)) + + pad0 :: Int -> String -> String + pad0 n s = + let l = length s in + if l < n + then (replicate (n-l) '0') ++ s + else s + toHex :: Char -> String + toHex c = map toUpper $ showHex (ord c) "" -- Turn a NormalWord like foo="bar $baz" into a series of constituent elements like [foo=,bar ,$baz] getWordParts (T_NormalWord _ l) = concatMap getWordParts l diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index b7a444d..102b908 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1759,7 +1759,7 @@ checkSshHereDoc _ (T_Redirecting _ redirs cmd) hasVariables = mkRegex "[`$]" checkHereDoc (T_FdRedirect _ _ (T_HereDoc id _ Unquoted token tokens)) | not (all isConstant tokens) = - warn id 2087 $ "Quote '" ++ token ++ "' to make here document expansions happen on the server side rather than on the client." + warn id 2087 $ "Quote '" ++ (e4m token) ++ "' to make here document expansions happen on the server side rather than on the client." checkHereDoc _ = return () checkSshHereDoc _ _ = return () @@ -2694,7 +2694,7 @@ checkUnpassedInFunctions params root = suggestParams (name, _, thing) = info (getId thing) 2119 $ - "Use " ++ name ++ " \"$@\" if function's $1 should mean script's $1." + "Use " ++ (e4m name) ++ " \"$@\" if function's $1 should mean script's $1." warnForDeclaration func name = warn (getId func) 2120 $ name ++ " references arguments, but none are ever passed." diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index aa4edd4..36f32a5 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -900,7 +900,7 @@ checkWhileGetoptsCase = CommandCheck (Exactly "getopts") f notRequested = Map.difference handledMap requestedMap warnUnhandled optId caseId str = - warn caseId 2213 $ "getopts specified -" ++ str ++ ", but it's not handled by this 'case'." + warn caseId 2213 $ "getopts specified -" ++ (e4m str) ++ ", but it's not handled by this 'case'." warnRedundant (Just str, expr) | str `notElem` ["*", ":", "?"] = diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 5634d95..b70e4f4 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -1904,14 +1904,14 @@ readPendingHereDocs = do debugHereDoc tokenId endToken doc | endToken `isInfixOf` doc = let lookAt line = when (endToken `isInfixOf` line) $ - parseProblemAtId tokenId ErrorC 1042 ("Close matches include '" ++ line ++ "' (!= '" ++ endToken ++ "').") + parseProblemAtId tokenId ErrorC 1042 ("Close matches include '" ++ (e4m line) ++ "' (!= '" ++ (e4m endToken) ++ "').") in do - parseProblemAtId tokenId ErrorC 1041 ("Found '" ++ endToken ++ "' further down, but not on a separate line.") + parseProblemAtId tokenId ErrorC 1041 ("Found '" ++ (e4m endToken) ++ "' further down, but not on a separate line.") mapM_ lookAt (lines doc) | map toLower endToken `isInfixOf` map toLower doc = - parseProblemAtId tokenId ErrorC 1043 ("Found " ++ endToken ++ " further down, but with wrong casing.") + parseProblemAtId tokenId ErrorC 1043 ("Found " ++ (e4m endToken) ++ " further down, but with wrong casing.") | otherwise = - parseProblemAtId tokenId ErrorC 1044 ("Couldn't find end token `" ++ endToken ++ "' in the here document.") + parseProblemAtId tokenId ErrorC 1044 ("Couldn't find end token `" ++ (e4m endToken) ++ "' in the here document.") readFilename = readNormalWord @@ -3168,7 +3168,7 @@ readConfigFile filename = do let line = "line " ++ (show . sourceLine $ errorPos err) suggestion = getStringFromParsec $ errorMessages err in - "Failed to process " ++ filename ++ ", " ++ line ++ ": " + "Failed to process " ++ (e4m filename) ++ ", " ++ line ++ ": " ++ suggestion prop_readConfigKVs1 = isOk readConfigKVs "disable=1234" From 2e5c56b27034492134be1538c1e1e6a533ca791a Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 31 Dec 2020 13:19:14 -0800 Subject: [PATCH 425/763] Parse heredocs correctly with carriage returns (fixes #2103) --- CHANGELOG.md | 1 + src/ShellCheck/Parser.hs | 10 ++++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 596db82..ff36bb2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ is still purely cosmetic and does not allow ShellCheck to continue. - Improved error reporting for trailing tokens after ]/]] and compound commands - `#!/usr/bin/env -S shell` is now handled correctly +- Here docs with \r are now parsed correctly and give better warnings ### Changed - Assignments are now parsed to spec, without leniency for leading $ or spaces diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index b70e4f4..afe6262 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -123,8 +123,10 @@ readUnicodeQuote = do return $ T_Literal id [c] carriageReturn = do - parseNote ErrorC 1017 "Literal carriage return. Run script through tr -d '\\r' ." + pos <- getPosition char '\r' + parseProblemAt pos ErrorC 1017 "Literal carriage return. Run script through tr -d '\\r' ." + return '\r' almostSpace = choice [ @@ -1761,6 +1763,8 @@ prop_readHereDoc17= isWarning readScript "cat <<- ' foo'\nbar\n foo\n foo\n" prop_readHereDoc18= isOk readScript "cat <<'\"foo'\nbar\n\"foo\n" prop_readHereDoc20= isWarning readScript "cat << foo\n foo\n()\nfoo\n" prop_readHereDoc21= isOk readScript "# shellcheck disable=SC1039\ncat << foo\n foo\n()\nfoo\n" +prop_readHereDoc22 = isWarning readScript "cat << foo\r\ncow\r\nfoo\r\n" +prop_readHereDoc23 = isNotOk readScript "cat << foo \r\ncow\r\nfoo\r\n" readHereDoc = called "here document" $ do pos <- getPosition try $ string "<<" @@ -1789,7 +1793,9 @@ readHereDoc = called "here document" $ do -- Fun fact: bash considers << foo"" quoted, but not << <("foo"). readToken = do str <- readStringForParser readNormalWord - return $ unquote str + -- A here doc actually works with \r\n because the \r becomes part of the token + crstr <- (carriageReturn >> (return $ str ++ "\r")) <|> return str + return $ unquote crstr readPendingHereDocs = do docs <- popPendingHereDocs From c5756760cbadf48e501896641f20f1bda9f17420 Mon Sep 17 00:00:00 2001 From: freddii Date: Tue, 5 Jan 2021 14:08:03 +0100 Subject: [PATCH 426/763] fixed typing mistakes in changelog --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff36bb2..2d7a1ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -164,7 +164,7 @@ - SC2204/SC2205: Warn about `( -z foo )` and `( foo -eq bar )` - SC2200/SC2201: Warn about brace expansion in [/[[ - SC2198/SC2199: Warn about arrays in [/[[ -- SC2196/SC2197: Warn about deprected egrep/fgrep +- SC2196/SC2197: Warn about deprecated egrep/fgrep - SC2195: Warn about unmatchable case branches - SC2194: Warn about constant 'case' statements - SC2193: Warn about `[[ file.png == *.mp3 ]]` and other unmatchables @@ -181,7 +181,7 @@ ### Fixed - `-c` no longer suggested when using `grep -o | wc` - Comments and whitespace are now allowed before filewide directives -- Here doc delimters with esoteric quoting like `foo""` are now handled +- Here doc delimiters with esoteric quoting like `foo""` are now handled - SC2095 about `ssh` in while read loops is now suppressed when using `-n` - `%(%Y%M%D)T` now recognized as a single formatter in `printf` checks - `grep -F` now suppresses regex related suggestions From dff8f9492a153b4ad8ac7d085136ce532e8ea081 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 5 Jan 2021 10:07:39 -0800 Subject: [PATCH 427/763] Improve SC2283 message and position --- src/ShellCheck/Analytics.hs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 102b908..8ab7f12 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -4236,16 +4236,22 @@ checkSecondArgIsComparison _ t = case argString of '=':'=':'=':'=':_ -> Nothing -- Don't warn about `echo ======` and such '+':'=':_ -> - return $ err (getId t) 2285 $ + return $ err (headId t) 2285 $ "Remove spaces around += to assign (or quote '+=' if literal)." '=':'=':_ -> return $ err (getId t) 2284 $ "Use [ x = y ] to compare values (or quote '==' if literal)." '=':_ -> - return $ err (getId t) 2283 $ - "Use [ ] to compare values, or remove spaces around = to assign (or quote '=' if literal)." + return $ err (headId arg) 2283 $ + "Remove spaces around = to assign (or use [ ] to compare, or quote '=' if literal)." _ -> Nothing _ -> return () + where + -- We don't pinpoint exactly, but this helps cases like foo =$bar + headId t = + case t of + T_NormalWord _ (x:_) -> getId x + _ -> getId t return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) From 99e9d5c54b0a3188b9c2b2ae09276429eea39b78 Mon Sep 17 00:00:00 2001 From: Kir Kolyshkin Date: Wed, 27 Jan 2021 16:21:44 -0800 Subject: [PATCH 428/763] Whitelist podman for SC2016 about '$var' Same as 08d2eef4111afde4 but for podman. Fixes https://github.com/koalaman/shellcheck/issues/2057 Signed-off-by: Kir Kolyshkin --- src/ShellCheck/Analytics.hs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 8ab7f12..a9ce515 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1015,6 +1015,7 @@ checkSingleQuotedVariables params t@(T_SingleQuoted id s) = ,"alias" ,"sudo" -- covering "sudo sh" and such ,"docker" -- like above + ,"podman" ,"dpkg-query" ,"jq" -- could also check that user provides --arg ,"rename" From 2e59eba6ebcafa029076a37cdea09521860ee01d Mon Sep 17 00:00:00 2001 From: Austin English Date: Fri, 5 Feb 2021 19:56:44 -0600 Subject: [PATCH 429/763] add support for `/bin/busybox sh` shebang --- src/ShellCheck/Analytics.hs | 6 ++++++ src/ShellCheck/Parser.hs | 4 +++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 8500a7b..cb02eae 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -590,6 +590,12 @@ prop_checkShebang9 = verifyNotTree checkShebang "# shellcheck shell=sh\ntrue" prop_checkShebang10= verifyNotTree checkShebang "#!foo\n# shellcheck shell=sh ignore=SC2239\ntrue" prop_checkShebang11= verifyTree checkShebang "#!/bin/sh/\ntrue" prop_checkShebang12= verifyTree checkShebang "#!/bin/sh/ -xe\ntrue" +prop_checkShebang13= verifyTree checkShebang "#!/bin/busybox sh" +prop_checkShebang14= verifyTree checkShebang "#!/bin/busybox sh\n# shellcheck shell=sh\n" +prop_checkShebang15= verifyNotTree checkShebang "#!/bin/busybox sh\n# shellcheck shell=dash\n" +prop_checkShebang16= verifyTree checkShebang "#!/bin/busybox ash" +prop_checkShebang17= verifyNotTree checkShebang "#!/bin/busybox ash\n# shellcheck shell=dash\n" +prop_checkShebang18= verifyNotTree checkShebang "#!/bin/busybox ash\n# shellcheck shell=sh\n" checkShebang params (T_Annotation _ list t) = if any isOverride list then [] else checkShebang params t where diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 03865af..a54bbc3 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -3238,7 +3238,9 @@ readScriptFile sourced = do (first:second:_) -> if basename first == "env" then second - else basename first + else if basename first == "busybox" + then second + else basename first verifyShebang pos s = do case isValidShell s of From 8bb5e01401ade0962727d9001ba34cfbf10937d6 Mon Sep 17 00:00:00 2001 From: Claudio Bley Date: Fri, 12 Feb 2021 10:44:18 +0100 Subject: [PATCH 430/763] Allow `env` to have flags and variables in shebang The `env` command has a `-S,--split-string` option which enables having arguments for the command in a shebang. Also, one could use variable assignments for the command since `env` treats only the first word without a `=` character as the command to run. Fixes #2143. --- src/ShellCheck/Parser.hs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 03865af..9f5208e 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -3190,6 +3190,7 @@ prop_readScript2 = isWarning readScript "#!/bin/bash\r\necho hello world\n" prop_readScript3 = isWarning readScript "#!/bin/bash\necho hello\xA0world" prop_readScript4 = isWarning readScript "#!/usr/bin/perl\nfoo=(" prop_readScript5 = isOk readScript "#!/bin/bash\n#This is an empty script\n\n" +prop_readScript6 = isOk readScript "#!/usr/bin/env -S X=FOO bash\n#This is an empty script\n\n" readScriptFile sourced = do start <- startSpan pos <- getPosition @@ -3231,13 +3232,14 @@ readScriptFile sourced = do where basename s = reverse . takeWhile (/= '/') . reverse $ s + skipFlags = dropWhile ("-" `isPrefixOf`) getShell sb = case words sb of [] -> "" [x] -> basename x - (first:second:_) -> + (first:args) -> if basename first == "env" - then second + then fromMaybe "" $ find (notElem '=') $ skipFlags args else basename first verifyShebang pos s = do From d6bb8fc0d8aa40e72b2b894b16d7eacdafcf94a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Moritz=20R=C3=B6hrich?= Date: Sun, 14 Feb 2021 19:13:29 +0100 Subject: [PATCH 431/763] Error on backslash in comment #2132 - Report error in case of a backspace in a comment Backspaces in comments are no good. In most cases they are the result of commenting out a longer line, that was broken down. This usually results in the shell treating the following lines as their own commands on their own lines instead of as parts of the longer, broken down line. --- src/ShellCheck/Parser.hs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 03865af..a999b23 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -1039,9 +1039,14 @@ readComment = do unexpecting "shellcheck annotation" readAnnotationPrefix readAnyComment +prop_readAnyComment = isOk readAnyComment "# Comment" +prop_readAnyComment1 = not $ isOk readAnyComment "# Comment \\\n" readAnyComment = do char '#' - many $ noneOf "\r\n" + comment <- many $ noneOf "\\\r\n" + bs <- many $ oneOf "\\" + unless (null bs) (fail "Backslash in or directly after comment") + return comment prop_readNormalWord = isOk readNormalWord "'foo'\"bar\"{1..3}baz$(lol)" prop_readNormalWord2 = isOk readNormalWord "foo**(foo)!!!(@@(bar))" From b9b6975bfa8d57dc5c332587f5e46aec0ea9fb74 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 11 Feb 2021 17:18:43 -0800 Subject: [PATCH 432/763] Brand New Build! Features Linux x86_64 docker builds for all archs --- .compile_binaries | 73 ------------------- .github/workflows/build.yml | 124 ++++++++++++++++++++++++++++++++ .github_deploy | 35 +-------- .multi_arch_docker | 9 +-- .prepare_deploy | 36 ++++++---- .travis.yml | 97 ------------------------- Dockerfile | 29 -------- build/README.md | 13 ++++ build/build_builder | 12 ++++ build/darwin.x86_64/Dockerfile | 31 ++++++++ build/darwin.x86_64/build | 14 ++++ build/darwin.x86_64/tag | 1 + build/linux.aarch64/Dockerfile | 30 ++++++++ build/linux.aarch64/build | 15 ++++ build/linux.aarch64/tag | 1 + build/linux.armv6hf/Dockerfile | 59 +++++++++++++++ build/linux.armv6hf/build | 16 +++++ build/linux.armv6hf/tag | 1 + build/linux.x86_64/Dockerfile | 26 +++++++ build/linux.x86_64/build | 15 ++++ build/linux.x86_64/tag | 1 + build/run_builder | 30 ++++++++ build/windows.x86_64/Dockerfile | 27 +++++++ build/windows.x86_64/build | 19 +++++ build/windows.x86_64/tag | 1 + 25 files changed, 461 insertions(+), 254 deletions(-) delete mode 100755 .compile_binaries create mode 100644 .github/workflows/build.yml delete mode 100644 .travis.yml delete mode 100644 Dockerfile create mode 100644 build/README.md create mode 100755 build/build_builder create mode 100644 build/darwin.x86_64/Dockerfile create mode 100755 build/darwin.x86_64/build create mode 100644 build/darwin.x86_64/tag create mode 100644 build/linux.aarch64/Dockerfile create mode 100755 build/linux.aarch64/build create mode 100644 build/linux.aarch64/tag create mode 100644 build/linux.armv6hf/Dockerfile create mode 100755 build/linux.armv6hf/build create mode 100644 build/linux.armv6hf/tag create mode 100644 build/linux.x86_64/Dockerfile create mode 100755 build/linux.x86_64/build create mode 100644 build/linux.x86_64/tag create mode 100755 build/run_builder create mode 100644 build/windows.x86_64/Dockerfile create mode 100755 build/windows.x86_64/build create mode 100644 build/windows.x86_64/tag diff --git a/.compile_binaries b/.compile_binaries deleted file mode 100755 index 95939ae..0000000 --- a/.compile_binaries +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/bash - -build_linux() { - # Linux Docker image - name="$DOCKER_BASE" - DOCKER_BUILDS="$DOCKER_BUILDS $name" - docker build -t "$name:current" . - docker run "$name:current" --version - printf '%s\n' "#!/bin/sh" "echo 'hello world'" > myscript - docker run -v "$PWD:/mnt" "$name:current" myscript - - # Copy static executable from docker image - id=$(docker create "$name:current") - docker cp "$id:/bin/shellcheck" "shellcheck" - docker rm "$id" - ls -l shellcheck - ./shellcheck myscript - for tag in $TAGS - do - cp "shellcheck" "deploy/shellcheck-$tag.linux-x86_64"; - done -} - -build_aarch64() { - # Linux aarch64 static executable - docker run -v "$PWD:/mnt" koalaman/aarch64-builder 'buildsc' - for tag in $TAGS - do - cp "shellcheck" "deploy/shellcheck-$tag.linux-aarch64" - done -} - - -build_armv6hf() { - # Linux armv6hf static executable - docker run -v "$PWD:/mnt" koalaman/armv6hf-builder -c 'compile-shellcheck' - for tag in $TAGS - do - cp "shellcheck" "deploy/shellcheck-$tag.linux-armv6hf"; - done -} - -build_windows() { - # Windows .exe - docker run -v "$PWD:/appdata" koalaman/winghc cuib - for tag in $TAGS - do - cp "dist/build/ShellCheck/shellcheck.exe" "deploy/shellcheck-$tag.exe"; - done -} - -build_osx() { - # Darwin x86_64 executable - brew update - brew install cabal-install pandoc gnu-tar - sudo ln -sf /usr/local/bin/gsha512sum /usr/local/bin/sha512sum - sudo ln -sf /usr/local/bin/gtar /usr/local/bin/tar - export PATH="/usr/local/bin:$PATH" - - cabal update - cabal install --dependencies-only - cabal build shellcheck - - # Cabal 3 no longer has a predictable output path - path="$(find . -name 'shellcheck' -type f -perm +111)" - [[ -e "$path" ]] - - for tag in $TAGS - do - cp "$path" "deploy/shellcheck-$tag.darwin-x86_64"; - done -} - diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..cd9fca0 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,124 @@ +name: Build Lol + +# Run this workflow every time a new commit pushed to your repository +on: push + +jobs: + package_source: + name: Package Source Code + runs-on: ubuntu-latest + steps: + - name: Install Dependencies + run: | + sudo apt-get update + sudo apt-mark manual ghc # Don't bother installing ghc just to tar up source + sudo apt-get install cabal-install + + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Package Source + run: | + mkdir source + cabal sdist + mv dist/*.tar.gz source/source.tar.gz + + - name: Deduce tags + run: | + exec > source/tags + echo "latest" + if tag=$(git describe --exact-match --tags) + then + echo "stable" + echo "$tag" + fi + + - name: Upload artifact + uses: actions/upload-artifact@v2 + with: + name: source + path: source/ + + build_source: + name: Build Source Code + needs: package_source + strategy: + matrix: + build: [linux.x86_64, linux.aarch64, linux.armv6hf, darwin.x86_64, windows.x86_64] + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Download artifacts + uses: actions/download-artifact@v2 + + - name: Build source + run: | + mkdir -p bin + mkdir -p bin/${{matrix.build}} + ( cd bin && ../build/run_builder ../source/source.tar.gz ../build/${{matrix.build}} ) + + - name: Upload artifact + uses: actions/upload-artifact@v2 + with: + name: bin + path: bin/ + + package_binary: + name: Package Binaries + needs: build_source + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Download artifacts + uses: actions/download-artifact@v2 + + - name: Work around GitHub permissions bug + run: chmod +x bin/*/shellcheck* + + - name: Package binaries + run: | + export TAGS="$(cat source/tags)" + mkdir -p deploy + cp -r bin/* deploy + cd deploy + ../.prepare_deploy + rm -rf */ README* LICENSE* + + - name: Upload artifact + uses: actions/upload-artifact@v2 + with: + name: deploy + path: deploy/ + + deploy: + name: Deploy binaries + needs: package_binary + runs-on: ubuntu-latest + environment: Deploy + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Download artifacts + uses: actions/download-artifact@v2 + + - name: Upload to GitHub + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + export TAGS="$(cat source/tags)" + ./.github_deploy + + - name: Upload to Docker Hub + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + DOCKER_EMAIL: ${{ secrets.DOCKER_EMAIL }} + DOCKER_BASE: ${{ secrets.DOCKER_USERNAME }}/shellcheck + run: | + export TAGS="$(cat source/tags)" + ( source ./.multi_arch_docker && set -eux && multi_arch_docker::main ) diff --git a/.github_deploy b/.github_deploy index e5d6c3e..3de0ac2 100755 --- a/.github_deploy +++ b/.github_deploy @@ -2,39 +2,10 @@ set -x shopt -s extglob -if [[ "$TRAVIS_SECURE_ENV_VARS" != "true" ]] -then - echo >&2 "Missing TRAVIS_SECURE_ENV_VARS. Skipping GitHub deployment." - exit 0 -fi - -install_deps() { - version="2.7.0" # 2.14.1 fails to overwrite duplicates - case "$(uname)" in - Linux) - sudo apt-get update - sudo apt-get install curl - curl -L "https://github.com/github/hub/releases/download/v$version/hub-linux-amd64-$version.tgz" | tar xvz --strip-components=1 "hub-linux-amd64-$version/bin/hub" - ;; - Darwin) - curl -L "https://github.com/github/hub/releases/download/v$version/hub-darwin-amd64-$version.tgz" | tar xvz --strip-components=1 "hub-darwin-amd64-$version/bin/hub" - ;; - *) - echo "Unknown: $(uname)" - exit 1 - ;; - esac - - hub_path="$PWD/bin/hub" - hub() { - "$hub_path" "$@" - } -} -install_deps - export EDITOR="touch" # Sanity check +gh --version || exit 1 hub release show latest || exit 1 for tag in $TAGS @@ -51,8 +22,8 @@ do do [[ $file == *.@(xz|gz|zip) ]] || continue [[ $file == *"$tag"* ]] || continue - files+=(-a "$file") + files+=("$file") done - hub release edit "${files[@]}" "$tag" || exit 1 + gh release upload "$tag" "${files[@]}" --clobber || exit 1 done diff --git a/.multi_arch_docker b/.multi_arch_docker index 294fed2..a9f7401 100755 --- a/.multi_arch_docker +++ b/.multi_arch_docker @@ -1,12 +1,6 @@ #!/bin/bash # This script builds and deploys multi-architecture docker images from the -# binaries previously built and deployed to GCS by the Travis pipeline. - -if [[ "$TRAVIS_SECURE_ENV_VARS" != "true" ]] -then - echo >&2 "Missing TRAVIS_SECURE_ENV_VARS. Skipping Docker builds." - exit 0 -fi +# binaries previously built and deployed to GitHub. function multi_arch_docker::install_docker_buildx() { # Install up-to-date version of docker, with buildx support. @@ -108,6 +102,5 @@ function multi_arch_docker::main() { multi_arch_docker::install_docker_buildx multi_arch_docker::login_to_docker_hub multi_arch_docker::build_and_push_all - set +x multi_arch_docker::test_all } diff --git a/.prepare_deploy b/.prepare_deploy index a2ce361..9f39912 100755 --- a/.prepare_deploy +++ b/.prepare_deploy @@ -1,8 +1,9 @@ #!/bin/bash -# This script packages up Travis compiled binaries +# This script packages up compiled binaries set -ex shopt -s nullglob extglob -cd deploy + +ls -l cp ../LICENSE LICENSE.txt sed -e $'s/$/\r/' > README.txt << END @@ -22,26 +23,31 @@ This binary was compiled on $(date -u). $(git log -n 3) END -for file in ./*.exe +for dir in */ do - zip "${file%.*}.zip" README.txt LICENSE.txt "$file" + cp LICENSE.txt README.txt "$dir" done -for file in *.{linux,darwin}-* -do - base="${file%.*}" - ext="${file##*.}" - os="${ext%-*}" - arch="${ext##*-}" - cp "$file" "shellcheck" - tar -cJf "$base.$os.$arch.tar.xz" --transform="s:^:$base/:" README.txt LICENSE.txt shellcheck - rm "shellcheck" -done +echo "Tags are $TAGS" -rm !(*.xz|*.zip) +for tag in $TAGS +do + + for dir in windows.*/ + do + ( cd "$dir" && zip "../shellcheck-$tag.zip" * ) + done + + for dir in {linux,darwin}.*/ + do + base="${dir%/}" + ( cd "$dir" && tar -cJf "../shellcheck-$tag.$base.tar.xz" --transform="s:^:shellcheck-$tag/:" * ) + done +done for file in ./* do + [[ -f "$file" ]] || continue sha512sum "$file" > "$file.sha512sum" done diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index d7d202b..0000000 --- a/.travis.yml +++ /dev/null @@ -1,97 +0,0 @@ -language: shell -os: linux - -services: - - docker - -jobs: - include: - - stage: Build - # This must weirdly not have a dash, otherwise an empty job is created - env: BUILD=linux - workspaces: - create: - name: ws-linux - paths: deploy - - env: BUILD=windows - workspaces: - create: - name: ws-windows - paths: deploy - - env: BUILD=armv6hf - workspaces: - create: - name: ws-armv6hf - paths: deploy - - env: BUILD=aarch64 - workspaces: - create: - name: ws-aarch64 - paths: deploy - - env: BUILD=osx - os: osx - workspaces: - create: - name: ws-osx - paths: deploy - - - stage: Upload Artifacts to GitHub - workspaces: - use: - - ws-osx - - ws-linux - - ws-armv6hf - - ws-aarch64 - - ws-windows - script: - - ls -la ${CASHER_DIR}/ || true - # Kludge broken TravisCI workspaces - - tar -xvf ${CASHER_DIR}/ws-osx-fetch.tgz --strip-components=5 - - ls -la deploy - - ./.github_deploy - - - stage: Deploy docker image - # Deploy only for pushes to master branch, not other branches, not PRs. - if: type = push - script: - - source ./.multi_arch_docker - - set -ex; multi_arch_docker::main; set +x - -# This is in global context and runs for every stage that doesn't override it. -before_install: | - DOCKER_BASE="$DOCKER_USERNAME/shellcheck" - DOCKER_BUILDS="" - export TAGS="" - test "$TRAVIS_BRANCH" = master && TAGS="$TAGS latest" || true - test -n "$TRAVIS_TAG" && TAGS="$TAGS stable $TRAVIS_TAG" || true - echo "Tags are $TAGS" - -# This is in global context and runs for every stage that doesn't override it. -script: - - mkdir -p deploy - - source ./.compile_binaries - - ./striptests - - set -ex; build_"$BUILD"; set +x; - - ./.prepare_deploy - -# This is in global context and runs for every stage that doesn't override it. -after_failure: | - id - pwd - df -h - find . -name '*.log' -type f -exec grep "" /dev/null {} + - find . -ls - -# This is in global context and runs for every stage that doesn't override it. -deploy: - provider: gcs - skip_cleanup: true - access_key_id: GOOG7MDN7WEH6IIGBDCA - secret_access_key: - secure: Bcx2cT0/E2ikj7sdamVq52xlLZF9dz9ojGPtoKfPyQhkkZa+McVI4xgUSuyyoSxyKj77sofx2y8m6PJYYumT4g5hREV1tfeUkl0J2DQFMbGDYEt7kxVkXCxojNvhHwTzLFv0ezstrxWWxQm81BfQQ4U9lggRXtndAP4czZnOeHPINPSiue1QNwRAEw05r5UoIUJXy/5xyUrjIxn381pAs+gJqP2COeN9kTKYH53nS/AAws29RprfZFnPlo7xxWmcjRcdS5KPdGXI/c6tQp5zl2iTh510VC1PN2w1Wvnn/oNWhiNdqPyVDsojIX5+sS3nejzJA+KFMxXSBlyXIY3wPpS/MdscU79X6Q5f9ivsFfsm7gNBmxHUPNn0HAvU4ROT/CCE9j6jSbs5PC7QBo3CK4++jxAwE/pd9HUc2rs3k0ofx3rgveJ7txpy5yPKfwIIBi98kVKlC4w7dLvNTOfjW1Imt2yH87XTfsE0UIG9st1WII6s4l/WgBx2GuwKdt6+3QUYiAlCFckkxWi+fAvpHZUEL43Qxub5fN+ZV7Zib1n7opchH4QKGBb6/y0WaDCmtCfu0lppoe/TH6saOTjDFj67NJSElK6ZDxGZ3uw4R+ret2gm6WRKT2Oeub8J33VzSa7VkmFpMPrAAfPa9N1Z4ewBLoTmvxSg2A0dDrCdJio= - bucket: shellcheck-private - local_dir: deploy - on: - repo: koalaman/shellcheck - condition: $TRAVIS_BUILD_STAGE_NAME = Build - all_branches: true diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 2f8f79e..0000000 --- a/Dockerfile +++ /dev/null @@ -1,29 +0,0 @@ -# Build-only image -FROM ubuntu:18.04 AS build -USER root -WORKDIR /opt/shellCheck - -# Install OS deps -RUN apt-get update && apt-get install -y ghc cabal-install - -# Install Haskell deps -# (This is a separate copy/run so that source changes don't require rebuilding) -COPY ShellCheck.cabal ./ -RUN cabal update && cabal install --dependencies-only --ghc-options="-optlo-Os -split-sections" - -# Copy source and build it -COPY LICENSE shellcheck.hs ./ -COPY src src -RUN cabal build Paths_ShellCheck && \ - ghc -optl-static -optl-pthread -isrc -idist/build/autogen --make shellcheck -split-sections -optc-Wl,--gc-sections -optlo-Os && \ - strip --strip-all shellcheck - -RUN mkdir -p /out/bin && \ - cp shellcheck /out/bin/ - -# Resulting ShellCheck image -FROM scratch -LABEL maintainer="Vidar Holen " -WORKDIR /mnt -COPY --from=build /out / -ENTRYPOINT ["/bin/shellcheck"] diff --git a/build/README.md b/build/README.md new file mode 100644 index 0000000..eb745a0 --- /dev/null +++ b/build/README.md @@ -0,0 +1,13 @@ +This directory contains Dockerfiles for all builds. + +A build image will: + +* Run on Linux x86\_64 with vanilla Docker (no exceptions) +* Not contain any software that would restrict easy modification or copying +* Take a `cabal sdist` style tar.gz of the ShellCheck directory on stdin +* Output a tar.gz of artifacts on stdout, in a directory named for the arch + +This makes it simple to build any release without exotic hardware or software. + +An image can be built and tagged using `build_builder`, +and run on a source tarball using `run_builder`. diff --git a/build/build_builder b/build/build_builder new file mode 100755 index 0000000..b34b996 --- /dev/null +++ b/build/build_builder @@ -0,0 +1,12 @@ +#!/bin/sh +if [ $# -eq 0 ] +then + echo >&2 "No build image directories specified" + echo >&2 "Example: $0 build/*/" + exit 1 +fi + +for dir +do + ( cd "$dir" && docker build -t "$(cat tag)" . ) || exit 1 +done diff --git a/build/darwin.x86_64/Dockerfile b/build/darwin.x86_64/Dockerfile new file mode 100644 index 0000000..e7425fe --- /dev/null +++ b/build/darwin.x86_64/Dockerfile @@ -0,0 +1,31 @@ +# DIGEST:sha256:fa32af4677e2860a1c5950bc8c360f309e2a87e2ddfed27b642fddf7a6093b76 +FROM liushuyu/osxcross:latest + +ENV TARGET x86_64-apple-darwin18 +ENV TARGETNAME darwin.x86_64 + +# Build dependencies +USER root +ENV DEBIAN_FRONTEND noninteractive +RUN apt-get update && apt-get install -y ghc automake autoconf llvm curl + +# Build GHC +WORKDIR /ghc +RUN curl -L "https://downloads.haskell.org/~ghc/8.10.4/ghc-8.10.4-src.tar.xz" | tar xJ --strip-components=1 +RUN ./boot && ./configure --host x86_64-linux-gnu --build x86_64-linux-gnu --target "$TARGET" +RUN cp mk/flavours/quick-cross.mk mk/build.mk && make -j "$(nproc)" +RUN make install +RUN curl -L "https://downloads.haskell.org/~cabal/cabal-install-3.2.0.0/cabal-install-3.2.0.0-x86_64-unknown-linux.tar.xz" | tar xJv -C /usr/local/bin + +# Due to an apparent cabal bug, we specify our options directly to cabal +# It won't reuse caches if ghc-options are specified in ~/.cabal/config +ENV CABALOPTS "--with-ghc=$TARGET-ghc;--with-hc-pkg=$TARGET-ghc-pkg" + +# Prebuild the dependencies +RUN cabal update && IFS=';' && cabal install $CABALOPTS --lib Diff-0.4.0 base-compat-0.11.2 base-orphans-0.8.4 dlist-1.0 hashable-1.3.0.0 indexed-traversable-0.1.1 integer-logarithms-1.0.3.1 primitive-0.7.1.0 regex-base-0.94.0.0 splitmix-0.1.0.3 tagged-0.8.6.1 th-abstraction-0.4.2.0 transformers-compat-0.6.6 base-compat-batteries-0.11.2 time-compat-1.9.5 unordered-containers-0.2.13.0 data-fix-0.3.1 vector-0.12.2.0 scientific-0.3.6.2 regex-tdfa-1.3.1.0 random-1.2.0 distributive-0.6.2.1 attoparsec-0.13.2.5 uuid-types-1.0.3 comonad-5.0.8 bifunctors-5.5.10 assoc-1.0.2 these-1.1.1.1 strict-0.4.0.1 aeson-1.5.5.1 + +# Copy the build script +COPY build /usr/bin + +WORKDIR /scratch +ENTRYPOINT ["/usr/bin/build"] diff --git a/build/darwin.x86_64/build b/build/darwin.x86_64/build new file mode 100755 index 0000000..03dfde7 --- /dev/null +++ b/build/darwin.x86_64/build @@ -0,0 +1,14 @@ +#!/bin/sh +set -xe +{ + tar xzv --strip-components=1 + ./striptests + mkdir "$TARGETNAME" + cabal update + ( IFS=';'; cabal build $CABALOPTS ) + find . -name shellcheck -type f -exec mv {} "$TARGETNAME/" \; + ls -l "$TARGETNAME" + "$TARGET-strip" -Sx "$TARGETNAME/shellcheck" + ls -l "$TARGETNAME" +} >&2 +tar czv "$TARGETNAME" diff --git a/build/darwin.x86_64/tag b/build/darwin.x86_64/tag new file mode 100644 index 0000000..237a65c --- /dev/null +++ b/build/darwin.x86_64/tag @@ -0,0 +1 @@ +koalaman/scbuilder-darwin-x86_64 diff --git a/build/linux.aarch64/Dockerfile b/build/linux.aarch64/Dockerfile new file mode 100644 index 0000000..8bf8d6f --- /dev/null +++ b/build/linux.aarch64/Dockerfile @@ -0,0 +1,30 @@ +FROM ubuntu:20.04 + +ENV TARGET aarch64-linux-gnu +ENV TARGETNAME linux.aarch64 + +# Build dependencies +USER root +ENV DEBIAN_FRONTEND noninteractive +RUN apt-get update && apt-get install -y ghc automake autoconf build-essential llvm curl qemu-user-static gcc-$TARGET + +# Build GHC +WORKDIR /ghc +RUN curl -L "https://downloads.haskell.org/~ghc/8.10.4/ghc-8.10.4-src.tar.xz" | tar xJ --strip-components=1 +RUN ./boot && ./configure --host x86_64-linux-gnu --build x86_64-linux-gnu --target "$TARGET" +RUN cp mk/flavours/quick-cross.mk mk/build.mk && make -j "$(nproc)" +RUN make install +RUN curl -L "https://downloads.haskell.org/~cabal/cabal-install-3.2.0.0/cabal-install-3.2.0.0-x86_64-unknown-linux.tar.xz" | tar xJv -C /usr/local/bin + +# Due to an apparent cabal bug, we specify our options directly to cabal +# It won't reuse caches if ghc-options are specified in ~/.cabal/config +ENV CABALOPTS "--ghc-options;-split-sections -optc-Os -optc-Wl,--gc-sections;--with-ghc=$TARGET-ghc;--with-hc-pkg=$TARGET-ghc-pkg" + +# Prebuild the dependencies +RUN cabal update && IFS=';' && cabal install $CABALOPTS --lib Diff-0.4.0 base-compat-0.11.2 base-orphans-0.8.4 dlist-1.0 hashable-1.3.0.0 indexed-traversable-0.1.1 integer-logarithms-1.0.3.1 primitive-0.7.1.0 regex-base-0.94.0.0 splitmix-0.1.0.3 tagged-0.8.6.1 th-abstraction-0.4.2.0 transformers-compat-0.6.6 base-compat-batteries-0.11.2 time-compat-1.9.5 unordered-containers-0.2.13.0 data-fix-0.3.1 vector-0.12.2.0 scientific-0.3.6.2 regex-tdfa-1.3.1.0 random-1.2.0 distributive-0.6.2.1 attoparsec-0.13.2.5 uuid-types-1.0.3 comonad-5.0.8 bifunctors-5.5.10 assoc-1.0.2 these-1.1.1.1 strict-0.4.0.1 aeson-1.5.5.1 + +# Copy the build script +COPY build /usr/bin + +WORKDIR /scratch +ENTRYPOINT ["/usr/bin/build"] diff --git a/build/linux.aarch64/build b/build/linux.aarch64/build new file mode 100755 index 0000000..1164dc1 --- /dev/null +++ b/build/linux.aarch64/build @@ -0,0 +1,15 @@ +#!/bin/sh +set -xe +{ + tar xzv --strip-components=1 + ./striptests + mkdir "$TARGETNAME" + cabal update + ( IFS=';'; cabal build $CABALOPTS --enable-executable-static ) + find . -name shellcheck -type f -exec mv {} "$TARGETNAME/" \; + ls -l "$TARGETNAME" + "$TARGET-strip" -s "$TARGETNAME/shellcheck" + ls -l "$TARGETNAME" + qemu-aarch64-static "$TARGETNAME/shellcheck" --version +} >&2 +tar czv "$TARGETNAME" diff --git a/build/linux.aarch64/tag b/build/linux.aarch64/tag new file mode 100644 index 0000000..6788e14 --- /dev/null +++ b/build/linux.aarch64/tag @@ -0,0 +1 @@ +koalaman/scbuilder-linux-aarch64 diff --git a/build/linux.armv6hf/Dockerfile b/build/linux.armv6hf/Dockerfile new file mode 100644 index 0000000..fee6d4c --- /dev/null +++ b/build/linux.armv6hf/Dockerfile @@ -0,0 +1,59 @@ +# I've again spent days trying to get a working armv6hf compiler going. +# God only knows how many recompilations of GCC, GHC, libraries, and +# ShellCheck itself, has gone into it. +# +# I tried Debian's toolchain. I tried my custom one built according to +# RPi `gcc -v`. I tried GHC9, glibc, musl, registerised vs not, but +# nothing has yielded an armv6hf binary that does not immediately +# segfault on qemu-arm-static or the RPi itself. +# +# I then tried the same but with armv7hf. Same story. +# +# Emulating the entire userspace with balenalib again? Very strange build +# failures where programs would fail to execute with > ~100 arguments. +# +# Finally, creating our own appears to work when using a custom QEmu +# patched to follow execve calls. +# +# PS: $100 bounty for getting a RPi1 compatible static build going +# with cross-compilation, similar to what the aarch64 build does. +# + +FROM ubuntu:20.04 + +ENV TARGETNAME linux.armv6hf + +# Build QEmu with execve follow support +USER root +ENV DEBIAN_FRONTEND noninteractive +RUN apt-get update +RUN apt-get install -y build-essential git ninja-build python3 pkg-config libglib2.0-dev libpixman-1-dev +WORKDIR /build +RUN git clone --depth 1 https://github.com/koalaman/qemu +RUN cd qemu && ./configure --static && cd build && ninja qemu-arm +RUN cp qemu/build/qemu-arm /build/qemu-arm-static +ENV QEMU_EXECVE 1 + +# Set up an armv6 userspace +WORKDIR / +RUN apt-get install -y debootstrap qemu-user-static +# We expect this to fail if the host doesn't have binfmt qemu support +RUN qemu-debootstrap --arch armhf bullseye pi http://mirrordirector.raspbian.org/raspbian || [ -e /pi/etc/issue ] +RUN cp /build/qemu-arm-static /pi/usr/bin/qemu-arm-static +RUN printf > /bin/pirun '%s\n' '#!/bin/sh' 'chroot /pi /usr/bin/qemu-arm-static /usr/bin/env "$@"' && chmod +x /bin/pirun +# If the debootstrap process didn't finish, continue it +RUN [ ! -e /pi/debootstrap ] || pirun '/debootstrap/debootstrap' --second-stage + +# Install deps in the chroot +RUN pirun apt-get update +RUN pirun apt-get install -y ghc cabal-install + +# Finally we can build the current dependencies. This takes hours. +ENV CABALOPTS "--ghc-options;-split-sections -optc-Os -optc-Wl,--gc-sections;--gcc-options;-Os -Wl,--gc-sections -ffunction-sections -fdata-sections" +RUN pirun cabal update +RUN IFS=";" && pirun cabal install --lib $CABALOPTS Diff-0.4.0 base-compat-0.11.2 base-orphans-0.8.4 dlist-1.0 hashable-1.3.1.0 indexed-traversable-0.1.1 integer-logarithms-1.0.3.1 primitive-0.7.1.0 regex-base-0.94.0.1 splitmix-0.1.0.3 tagged-0.8.6.1 th-abstraction-0.4.2.0 transformers-compat-0.6.6 base-compat-batteries-0.11.2 time-compat-1.9.5 unordered-containers-0.2.13.0 data-fix-0.3.1 vector-0.12.2.0 scientific-0.3.6.2 regex-tdfa-1.3.1.0 random-1.2.0 distributive-0.6.2.1 attoparsec-0.13.2.5 uuid-types-1.0.4 comonad-5.0.8 bifunctors-5.5.10 assoc-1.0.2 these-1.1.1.1 strict-0.4.0.1 aeson-1.5.6.0 + +# Copy the build script +WORKDIR /pi/scratch +COPY build /pi/usr/bin +ENTRYPOINT ["/bin/pirun", "/usr/bin/build"] diff --git a/build/linux.armv6hf/build b/build/linux.armv6hf/build new file mode 100755 index 0000000..4f2c7bc --- /dev/null +++ b/build/linux.armv6hf/build @@ -0,0 +1,16 @@ +#!/bin/sh +set -xe +cd /scratch +{ + tar xzv --strip-components=1 + ./striptests + mkdir "$TARGETNAME" + # This script does not cabal update because compiling anything new is slow + ( IFS=';'; cabal build $CABALOPTS --enable-executable-static ) + find . -name shellcheck -type f -exec mv {} "$TARGETNAME/" \; + ls -l "$TARGETNAME" + strip -s "$TARGETNAME/shellcheck" + ls -l "$TARGETNAME" + "$TARGETNAME/shellcheck" --version +} >&2 +tar czv "$TARGETNAME" diff --git a/build/linux.armv6hf/tag b/build/linux.armv6hf/tag new file mode 100644 index 0000000..9172c5c --- /dev/null +++ b/build/linux.armv6hf/tag @@ -0,0 +1 @@ +koalaman/scbuilder-linux-armv6hf diff --git a/build/linux.x86_64/Dockerfile b/build/linux.x86_64/Dockerfile new file mode 100644 index 0000000..f0ad16a --- /dev/null +++ b/build/linux.x86_64/Dockerfile @@ -0,0 +1,26 @@ +FROM ubuntu:20.04 + +ENV TARGETNAME linux.x86_64 + +# Install GHC and cabal +USER root +ENV DEBIAN_FRONTEND noninteractive +RUN apt-get update && apt-get install -y ghc curl xz-utils + +# So we'd like a later version of Cabal that supports --enable-executable-static, +# but we can't use Ubuntu 20.10 because coreutils has switched to new syscalls that +# the TravisCI kernel doesn't support. Download it manually. +RUN curl "https://downloads.haskell.org/~cabal/cabal-install-3.2.0.0/cabal-install-3.2.0.0-x86_64-unknown-linux.tar.xz" | tar xJv -C /usr/bin + +# Use ld.bfd instead of ld.gold due to +# x86_64-linux-gnu/libpthread.a(pthread_cond_init.o)(.note.stapsdt+0x14): error: +# relocation refers to local symbol "" [2], which is defined in a discarded section +ENV CABALOPTS "--ghc-options;-optl-Wl,-fuse-ld=bfd -split-sections -optc-Os -optc-Wl,--gc-sections" + +# Other archs pre-build dependencies here, but this one doesn't to detect ecosystem movement + +# Copy the build script +COPY build /usr/bin + +WORKDIR /scratch +ENTRYPOINT ["/usr/bin/build"] diff --git a/build/linux.x86_64/build b/build/linux.x86_64/build new file mode 100755 index 0000000..ba598e6 --- /dev/null +++ b/build/linux.x86_64/build @@ -0,0 +1,15 @@ +#!/bin/sh +set -xe +{ + tar xzv --strip-components=1 + ./striptests + mkdir "$TARGETNAME" + cabal update + ( IFS=';'; cabal build $CABALOPTS --enable-executable-static ) + find . -name shellcheck -type f -exec mv {} "$TARGETNAME/" \; + ls -l "$TARGETNAME" + strip -s "$TARGETNAME/shellcheck" + ls -l "$TARGETNAME" + "$TARGETNAME/shellcheck" --version +} >&2 +tar czv "$TARGETNAME" diff --git a/build/linux.x86_64/tag b/build/linux.x86_64/tag new file mode 100644 index 0000000..f0224de --- /dev/null +++ b/build/linux.x86_64/tag @@ -0,0 +1 @@ +koalaman/scbuilder-linux-x86_64 diff --git a/build/run_builder b/build/run_builder new file mode 100755 index 0000000..d6de27b --- /dev/null +++ b/build/run_builder @@ -0,0 +1,30 @@ +#!/bin/bash +if [ $# -lt 2 ] +then + echo >&2 "This script builds a source archive (as produced by cabal sdist)" + echo >&2 "Usage: $0 sourcefile.tar.gz builddir..." + exit 1 +fi + +file=$(realpath "$1") +shift + +if [ ! -e "$file" ] +then + echo >&2 "$file does not exist" + exit 1 +fi + +set -ex -o pipefail + +for dir +do + tagfile="$dir/tag" + if [ ! -e "$tagfile" ] + then + echo >&2 "$tagfile does not exist" + exit 2 + fi + + docker run -i "$(< "$tagfile")" < "$file" | tar xz +done diff --git a/build/windows.x86_64/Dockerfile b/build/windows.x86_64/Dockerfile new file mode 100644 index 0000000..3eb20fa --- /dev/null +++ b/build/windows.x86_64/Dockerfile @@ -0,0 +1,27 @@ +FROM ubuntu:20.04 + +ENV TARGETNAME windows.x86_64 + +# We don't need wine32, even though it complains +USER root +ENV DEBIAN_FRONTEND noninteractive +RUN apt-get update && apt-get install -y curl busybox wine + +# Fetch Windows version, will be available under z:\haskell +WORKDIR /haskell +RUN curl -L "https://downloads.haskell.org/~ghc/8.10.4/ghc-8.10.4-x86_64-unknown-mingw32.tar.xz" | tar xJ --strip-components=1 +WORKDIR /haskell/bin +RUN curl -L "https://downloads.haskell.org/~cabal/cabal-install-3.2.0.0/cabal-install-3.2.0.0-x86_64-unknown-mingw32.zip" | busybox unzip - +RUN curl -L "https://curl.se/windows/dl-7.75.0/curl-7.75.0-win64-mingw.zip" | busybox unzip - && mv curl-7.75.0-win64-mingw/bin/* . +ENV WINEPATH /haskell/bin + +# It's unknown whether Cabal on Windows suffers from the same issue +# that necessitated this but I don't care enough to find out +ENV CABALOPTS "--ghc-options;-split-sections -optc-Os -optc-Wl,--gc-sections" + +# Precompile some deps to speed up later builds. This list is just copied from `cabal build` +RUN wine /haskell/bin/cabal.exe update && IFS=';' && wine /haskell/bin/cabal.exe install $CABALOPTS --lib Diff-0.4.0 base-compat-0.11.2 base-orphans-0.8.4 dlist-1.0 hashable-1.3.0.0 indexed-traversable-0.1.1 integer-logarithms-1.0.3.1 primitive-0.7.1.0 regex-base-0.94.0.0 splitmix-0.1.0.3 tagged-0.8.6.1 th-abstraction-0.4.2.0 transformers-compat-0.6.6 base-compat-batteries-0.11.2 time-compat-1.9.5 unordered-containers-0.2.13.0 data-fix-0.3.1 vector-0.12.2.0 scientific-0.3.6.2 regex-tdfa-1.3.1.0 random-1.2.0 distributive-0.6.2.1 attoparsec-0.13.2.5 uuid-types-1.0.3 comonad-5.0.8 bifunctors-5.5.10 assoc-1.0.2 these-1.1.1.1 strict-0.4.0.1 aeson-1.5.5.1 + +COPY build /usr/bin +WORKDIR /scratch +ENTRYPOINT ["/usr/bin/build"] diff --git a/build/windows.x86_64/build b/build/windows.x86_64/build new file mode 100755 index 0000000..bedb870 --- /dev/null +++ b/build/windows.x86_64/build @@ -0,0 +1,19 @@ +#!/bin/sh +cabal() { + wine /haskell/bin/cabal.exe "$@" +} + +set -xe +{ + tar xzv --strip-components=1 + ./striptests + mkdir "$TARGETNAME" + cabal update + ( IFS=';'; cabal build $CABALOPTS ) + find dist*/ -name shellcheck.exe -type f -ls -exec mv {} "$TARGETNAME/" \; + ls -l "$TARGETNAME" + wine "/haskell/mingw/bin/strip.exe" -s "$TARGETNAME/shellcheck.exe" + ls -l "$TARGETNAME" + wine "$TARGETNAME/shellcheck.exe" --version +} >&2 +tar czv "$TARGETNAME" diff --git a/build/windows.x86_64/tag b/build/windows.x86_64/tag new file mode 100644 index 0000000..a85921b --- /dev/null +++ b/build/windows.x86_64/tag @@ -0,0 +1 @@ +koalaman/scbuilder-windows-x86_64 From 8442695b736dd10a3cf8e8be3b55986a29082e5a Mon Sep 17 00:00:00 2001 From: Viacheslav Vasilyev Date: Sat, 6 Mar 2021 14:19:08 +0200 Subject: [PATCH 433/763] suppress ntlm error messages in Windows build when building for windows there are many error message like below ``` 003a:err:winediag:SECUR32_initNTLMSP ntlm_auth was not found or is outdated. Make sure that ntlm_auth >= 3.0.25 is in your path. Usually, you can find it in the winbind package of your distribution. ``` --- build/windows.x86_64/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/windows.x86_64/Dockerfile b/build/windows.x86_64/Dockerfile index 3eb20fa..c5fa07b 100644 --- a/build/windows.x86_64/Dockerfile +++ b/build/windows.x86_64/Dockerfile @@ -5,7 +5,7 @@ ENV TARGETNAME windows.x86_64 # We don't need wine32, even though it complains USER root ENV DEBIAN_FRONTEND noninteractive -RUN apt-get update && apt-get install -y curl busybox wine +RUN apt-get update && apt-get install -y curl busybox wine winbind # Fetch Windows version, will be available under z:\haskell WORKDIR /haskell From 88cd21fd0f679d96b44abbbb343f1d9eb66b2c28 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 22 Feb 2021 21:28:14 -0800 Subject: [PATCH 434/763] Fix missing +x with new cabal and use previous release deps for caching --- .github/workflows/build.yml | 4 ++-- build/darwin.x86_64/Dockerfile | 2 +- build/darwin.x86_64/build | 2 +- build/linux.aarch64/Dockerfile | 2 +- build/linux.aarch64/build | 2 +- build/linux.armv6hf/Dockerfile | 2 +- build/linux.armv6hf/build | 2 +- build/linux.x86_64/build | 2 +- build/windows.x86_64/Dockerfile | 4 ++-- build/windows.x86_64/build | 2 +- 10 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cd9fca0..707b9a2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,4 +1,4 @@ -name: Build Lol +name: Build ShellCheck # Run this workflow every time a new commit pushed to your repository on: push @@ -21,7 +21,7 @@ jobs: run: | mkdir source cabal sdist - mv dist/*.tar.gz source/source.tar.gz + mv dist-newstyle/sdist/*.tar.gz source/source.tar.gz - name: Deduce tags run: | diff --git a/build/darwin.x86_64/Dockerfile b/build/darwin.x86_64/Dockerfile index e7425fe..ecd1cad 100644 --- a/build/darwin.x86_64/Dockerfile +++ b/build/darwin.x86_64/Dockerfile @@ -22,7 +22,7 @@ RUN curl -L "https://downloads.haskell.org/~cabal/cabal-install-3.2.0.0/cabal-in ENV CABALOPTS "--with-ghc=$TARGET-ghc;--with-hc-pkg=$TARGET-ghc-pkg" # Prebuild the dependencies -RUN cabal update && IFS=';' && cabal install $CABALOPTS --lib Diff-0.4.0 base-compat-0.11.2 base-orphans-0.8.4 dlist-1.0 hashable-1.3.0.0 indexed-traversable-0.1.1 integer-logarithms-1.0.3.1 primitive-0.7.1.0 regex-base-0.94.0.0 splitmix-0.1.0.3 tagged-0.8.6.1 th-abstraction-0.4.2.0 transformers-compat-0.6.6 base-compat-batteries-0.11.2 time-compat-1.9.5 unordered-containers-0.2.13.0 data-fix-0.3.1 vector-0.12.2.0 scientific-0.3.6.2 regex-tdfa-1.3.1.0 random-1.2.0 distributive-0.6.2.1 attoparsec-0.13.2.5 uuid-types-1.0.3 comonad-5.0.8 bifunctors-5.5.10 assoc-1.0.2 these-1.1.1.1 strict-0.4.0.1 aeson-1.5.5.1 +RUN cabal update && IFS=';' && cabal install --dependencies-only $CABALOPTS ShellCheck # Copy the build script COPY build /usr/bin diff --git a/build/darwin.x86_64/build b/build/darwin.x86_64/build index 03dfde7..53857e8 100755 --- a/build/darwin.x86_64/build +++ b/build/darwin.x86_64/build @@ -2,7 +2,7 @@ set -xe { tar xzv --strip-components=1 - ./striptests + chmod +x striptests && ./striptests mkdir "$TARGETNAME" cabal update ( IFS=';'; cabal build $CABALOPTS ) diff --git a/build/linux.aarch64/Dockerfile b/build/linux.aarch64/Dockerfile index 8bf8d6f..60537b3 100644 --- a/build/linux.aarch64/Dockerfile +++ b/build/linux.aarch64/Dockerfile @@ -21,7 +21,7 @@ RUN curl -L "https://downloads.haskell.org/~cabal/cabal-install-3.2.0.0/cabal-in ENV CABALOPTS "--ghc-options;-split-sections -optc-Os -optc-Wl,--gc-sections;--with-ghc=$TARGET-ghc;--with-hc-pkg=$TARGET-ghc-pkg" # Prebuild the dependencies -RUN cabal update && IFS=';' && cabal install $CABALOPTS --lib Diff-0.4.0 base-compat-0.11.2 base-orphans-0.8.4 dlist-1.0 hashable-1.3.0.0 indexed-traversable-0.1.1 integer-logarithms-1.0.3.1 primitive-0.7.1.0 regex-base-0.94.0.0 splitmix-0.1.0.3 tagged-0.8.6.1 th-abstraction-0.4.2.0 transformers-compat-0.6.6 base-compat-batteries-0.11.2 time-compat-1.9.5 unordered-containers-0.2.13.0 data-fix-0.3.1 vector-0.12.2.0 scientific-0.3.6.2 regex-tdfa-1.3.1.0 random-1.2.0 distributive-0.6.2.1 attoparsec-0.13.2.5 uuid-types-1.0.3 comonad-5.0.8 bifunctors-5.5.10 assoc-1.0.2 these-1.1.1.1 strict-0.4.0.1 aeson-1.5.5.1 +RUN cabal update && IFS=';' && cabal install --dependencies-only $CABALOPTS ShellCheck # Copy the build script COPY build /usr/bin diff --git a/build/linux.aarch64/build b/build/linux.aarch64/build index 1164dc1..f8001aa 100755 --- a/build/linux.aarch64/build +++ b/build/linux.aarch64/build @@ -2,7 +2,7 @@ set -xe { tar xzv --strip-components=1 - ./striptests + chmod +x striptests && ./striptests mkdir "$TARGETNAME" cabal update ( IFS=';'; cabal build $CABALOPTS --enable-executable-static ) diff --git a/build/linux.armv6hf/Dockerfile b/build/linux.armv6hf/Dockerfile index fee6d4c..bd5795c 100644 --- a/build/linux.armv6hf/Dockerfile +++ b/build/linux.armv6hf/Dockerfile @@ -51,7 +51,7 @@ RUN pirun apt-get install -y ghc cabal-install # Finally we can build the current dependencies. This takes hours. ENV CABALOPTS "--ghc-options;-split-sections -optc-Os -optc-Wl,--gc-sections;--gcc-options;-Os -Wl,--gc-sections -ffunction-sections -fdata-sections" RUN pirun cabal update -RUN IFS=";" && pirun cabal install --lib $CABALOPTS Diff-0.4.0 base-compat-0.11.2 base-orphans-0.8.4 dlist-1.0 hashable-1.3.1.0 indexed-traversable-0.1.1 integer-logarithms-1.0.3.1 primitive-0.7.1.0 regex-base-0.94.0.1 splitmix-0.1.0.3 tagged-0.8.6.1 th-abstraction-0.4.2.0 transformers-compat-0.6.6 base-compat-batteries-0.11.2 time-compat-1.9.5 unordered-containers-0.2.13.0 data-fix-0.3.1 vector-0.12.2.0 scientific-0.3.6.2 regex-tdfa-1.3.1.0 random-1.2.0 distributive-0.6.2.1 attoparsec-0.13.2.5 uuid-types-1.0.4 comonad-5.0.8 bifunctors-5.5.10 assoc-1.0.2 these-1.1.1.1 strict-0.4.0.1 aeson-1.5.6.0 +RUN IFS=";" && pirun cabal install --dependencies-only $CABALOPTS ShellCheck # Copy the build script WORKDIR /pi/scratch diff --git a/build/linux.armv6hf/build b/build/linux.armv6hf/build index 4f2c7bc..daa94d9 100755 --- a/build/linux.armv6hf/build +++ b/build/linux.armv6hf/build @@ -3,7 +3,7 @@ set -xe cd /scratch { tar xzv --strip-components=1 - ./striptests + chmod +x striptests && ./striptests mkdir "$TARGETNAME" # This script does not cabal update because compiling anything new is slow ( IFS=';'; cabal build $CABALOPTS --enable-executable-static ) diff --git a/build/linux.x86_64/build b/build/linux.x86_64/build index ba598e6..099f127 100755 --- a/build/linux.x86_64/build +++ b/build/linux.x86_64/build @@ -2,7 +2,7 @@ set -xe { tar xzv --strip-components=1 - ./striptests + chmod +x striptests && ./striptests mkdir "$TARGETNAME" cabal update ( IFS=';'; cabal build $CABALOPTS --enable-executable-static ) diff --git a/build/windows.x86_64/Dockerfile b/build/windows.x86_64/Dockerfile index 3eb20fa..b412c1c 100644 --- a/build/windows.x86_64/Dockerfile +++ b/build/windows.x86_64/Dockerfile @@ -19,8 +19,8 @@ ENV WINEPATH /haskell/bin # that necessitated this but I don't care enough to find out ENV CABALOPTS "--ghc-options;-split-sections -optc-Os -optc-Wl,--gc-sections" -# Precompile some deps to speed up later builds. This list is just copied from `cabal build` -RUN wine /haskell/bin/cabal.exe update && IFS=';' && wine /haskell/bin/cabal.exe install $CABALOPTS --lib Diff-0.4.0 base-compat-0.11.2 base-orphans-0.8.4 dlist-1.0 hashable-1.3.0.0 indexed-traversable-0.1.1 integer-logarithms-1.0.3.1 primitive-0.7.1.0 regex-base-0.94.0.0 splitmix-0.1.0.3 tagged-0.8.6.1 th-abstraction-0.4.2.0 transformers-compat-0.6.6 base-compat-batteries-0.11.2 time-compat-1.9.5 unordered-containers-0.2.13.0 data-fix-0.3.1 vector-0.12.2.0 scientific-0.3.6.2 regex-tdfa-1.3.1.0 random-1.2.0 distributive-0.6.2.1 attoparsec-0.13.2.5 uuid-types-1.0.3 comonad-5.0.8 bifunctors-5.5.10 assoc-1.0.2 these-1.1.1.1 strict-0.4.0.1 aeson-1.5.5.1 +# Precompile some deps to speed up later builds +RUN wine /haskell/bin/cabal.exe update && IFS=';' && wine /haskell/bin/cabal.exe install --lib --dependencies-only $CABALOPTS ShellCheck COPY build /usr/bin WORKDIR /scratch diff --git a/build/windows.x86_64/build b/build/windows.x86_64/build index bedb870..7bf186e 100755 --- a/build/windows.x86_64/build +++ b/build/windows.x86_64/build @@ -6,7 +6,7 @@ cabal() { set -xe { tar xzv --strip-components=1 - ./striptests + chmod +x striptests && ./striptests mkdir "$TARGETNAME" cabal update ( IFS=';'; cabal build $CABALOPTS ) From f02c297fddb9de8b45a5fedb912fb8c1f96b79f6 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 11 Mar 2021 23:04:17 -0800 Subject: [PATCH 435/763] Merge parser and analyzer shebang parsing --- src/ShellCheck/ASTLib.hs | 44 +++++++++++++++++++++++++++++++++++ src/ShellCheck/Analytics.hs | 2 +- src/ShellCheck/AnalyzerLib.hs | 15 ++---------- src/ShellCheck/Parser.hs | 18 +++----------- test/shellcheck.hs | 2 ++ 5 files changed, 52 insertions(+), 29 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index c5f1735..019fc3c 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -17,9 +17,11 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . -} +{-# LANGUAGE TemplateHaskell #-} module ShellCheck.ASTLib where import ShellCheck.AST +import ShellCheck.Regex import Control.Monad.Writer import Control.Monad @@ -31,6 +33,8 @@ import Data.Maybe import qualified Data.Map as Map import Numeric (showHex) +import Test.QuickCheck + arguments (T_SimpleCommand _ _ (cmd:args)) = args -- Is this a type of loop? @@ -672,3 +676,43 @@ isAnnotationIgnoringCode code t = where hasNum (DisableComment from to) = code >= from && code < to hasNum _ = False + +prop_executableFromShebang1 = executableFromShebang "/bin/sh" == "sh" +prop_executableFromShebang2 = executableFromShebang "/bin/bash" == "bash" +prop_executableFromShebang3 = executableFromShebang "/usr/bin/env ksh" == "ksh" +prop_executableFromShebang4 = executableFromShebang "/usr/bin/env -S foo=bar bash -x" == "bash" +prop_executableFromShebang5 = executableFromShebang "/usr/bin/env --split-string=bash -x" == "bash" +prop_executableFromShebang6 = executableFromShebang "/usr/bin/env --split-string=foo=bar bash -x" == "bash" +prop_executableFromShebang7 = executableFromShebang "/usr/bin/env --split-string bash -x" == "bash" +prop_executableFromShebang8 = executableFromShebang "/usr/bin/env --split-string foo=bar bash -x" == "bash" +prop_executableFromShebang9 = executableFromShebang "/usr/bin/env foo=bar dash" == "dash" +prop_executableFromShebang10 = executableFromShebang "/bin/busybox sh" == "ash" +prop_executableFromShebang11 = executableFromShebang "/bin/busybox ash" == "ash" + +-- Get the shell executable from a string like '/usr/bin/env bash' +executableFromShebang :: String -> String +executableFromShebang = shellFor + where + re = mkRegex "/env +(-S|--split-string=?)? *(.*)" + shellFor s | s `matches` re = + case matchRegex re s of + Just [flag, shell] -> fromEnvArgs (words shell) + _ -> "" + shellFor sb = + case words sb of + [] -> "" + [x] -> basename x + (first:second:args) | basename first == "busybox" -> + case basename second of + "sh" -> "ash" -- busybox sh is ash + x -> x + (first:args) | basename first == "env" -> + fromEnvArgs args + (first:_) -> basename first + + fromEnvArgs args = fromMaybe "" $ find (notElem '=') $ skipFlags args + basename s = reverse . takeWhile (/= '/') . reverse $ s + skipFlags = dropWhile ("-" `isPrefixOf`) + +return [] +runTests = $quickCheckAll diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 160ae95..9035e04 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -591,7 +591,7 @@ prop_checkShebang10= verifyNotTree checkShebang "#!foo\n# shellcheck shell=sh ig prop_checkShebang11= verifyTree checkShebang "#!/bin/sh/\ntrue" prop_checkShebang12= verifyTree checkShebang "#!/bin/sh/ -xe\ntrue" prop_checkShebang13= verifyTree checkShebang "#!/bin/busybox sh" -prop_checkShebang14= verifyTree checkShebang "#!/bin/busybox sh\n# shellcheck shell=sh\n" +prop_checkShebang14= verifyNotTree checkShebang "#!/bin/busybox sh\n# shellcheck shell=sh\n" prop_checkShebang15= verifyNotTree checkShebang "#!/bin/busybox sh\n# shellcheck shell=dash\n" prop_checkShebang16= verifyTree checkShebang "#!/bin/busybox ash" prop_checkShebang17= verifyNotTree checkShebang "#!/bin/busybox ash\n# shellcheck shell=dash\n" diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index d3b1134..5b97e87 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -240,6 +240,8 @@ prop_determineShell7 = determineShellTest "#! /bin/ash" == Dash prop_determineShell8 = determineShellTest' (Just Ksh) "#!/bin/sh" == Sh prop_determineShell9 = determineShellTest "#!/bin/env -S dash -x" == Dash prop_determineShell10 = determineShellTest "#!/bin/env --split-string= dash -x" == Dash +prop_determineShell11 = determineShellTest "#!/bin/busybox sh" == Dash -- busybox sh is a specific shell, not posix sh +prop_determineShell12 = determineShellTest "#!/bin/busybox ash" == Dash determineShellTest = determineShellTest' Nothing determineShellTest' fallbackShell = determineShell fallbackShell . fromJust . prRoot . pScript @@ -253,19 +255,6 @@ determineShell fallbackShell t = fromMaybe Bash $ headOrDefault (fromShebang s) [s | ShellOverride s <- annotations] fromShebang (T_Script _ (T_Literal _ s) _) = executableFromShebang s --- Given a string like "/bin/bash" or "/usr/bin/env dash", --- return the shell basename like "bash" or "dash" -executableFromShebang :: String -> String -executableFromShebang = shellFor - where - shellFor s | "/env " `isInfixOf` s = case matchRegex re s of - Just [flag, shell] -> shell - _ -> "" - shellFor s | ' ' `elem` s = shellFor $ takeWhile (/= ' ') s - shellFor s = reverse . takeWhile (/= '/') . reverse $ s - re = mkRegex "/env +(-S|--split-string=?)? *([^ ]*)" - - -- Given a root node, make a map from Id to parent Token. -- This is used to populate parentMap in Parameters getParentTree :: Token -> Map.Map Id Token diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 70ea05d..f32c20b 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -24,7 +24,7 @@ module ShellCheck.Parser (parseScript, runTests) where import ShellCheck.AST -import ShellCheck.ASTLib +import ShellCheck.ASTLib hiding (runTests) import ShellCheck.Data import ShellCheck.Interface @@ -3216,8 +3216,8 @@ readScriptFile sourced = do let ignoreShebang = shellAnnotationSpecified || shellFlagSpecified unless ignoreShebang $ - verifyShebang pos (getShell shebangString) - if ignoreShebang || isValidShell (getShell shebangString) /= Just False + verifyShebang pos (executableFromShebang shebangString) + if ignoreShebang || isValidShell (executableFromShebang shebangString) /= Just False then do commands <- withAnnotations annotations readCompoundListOrEmpty id <- endSpan start @@ -3231,18 +3231,6 @@ readScriptFile sourced = do return $ T_Script id shebang [] where - basename s = reverse . takeWhile (/= '/') . reverse $ s - skipFlags = dropWhile ("-" `isPrefixOf`) - getShell sb = - case words sb of - [] -> "" - [x] -> basename x - (first:args) | basename first == "env" -> - fromMaybe "" $ find (notElem '=') $ skipFlags args - (first:second:args) | basename first == "busybox" -> - second - (first:_) -> basename first - verifyShebang pos s = do case isValidShell s of Just True -> return () diff --git a/test/shellcheck.hs b/test/shellcheck.hs index ac84116..e463403 100644 --- a/test/shellcheck.hs +++ b/test/shellcheck.hs @@ -4,6 +4,7 @@ import Control.Monad import System.Exit import qualified ShellCheck.Analytics import qualified ShellCheck.AnalyzerLib +import qualified ShellCheck.ASTLib import qualified ShellCheck.Checker import qualified ShellCheck.Checks.Commands import qualified ShellCheck.Checks.Custom @@ -17,6 +18,7 @@ main = do results <- sequence [ ShellCheck.Analytics.runTests ,ShellCheck.AnalyzerLib.runTests + ,ShellCheck.ASTLib.runTests ,ShellCheck.Checker.runTests ,ShellCheck.Checks.Commands.runTests ,ShellCheck.Checks.Custom.runTests From 98952df35b834315c50c2ce6fe8f96516bf2a7ae Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 20 Mar 2021 13:58:37 -0700 Subject: [PATCH 436/763] Improve warnings on backslashes in comments --- CHANGELOG.md | 1 + src/ShellCheck/Parser.hs | 24 +++++++++++++++++------- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d7a1ac..9eec957 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## Git ### Added - `disable` directives can now be a range, e.g. `disable=SC3000-SC4000` +- SC1143: Warn about line continuations in comments - SC2259/SC2260: Warn when redirections override pipes - SC2261: Warn about multiple competing redirections - SC2262/SC2263: Warn about aliases declared and used in the same parsing unit diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 8db1991..3c16d5d 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -87,11 +87,23 @@ extglobStart = oneOf extglobStartChars unicodeDoubleQuotes = "\x201C\x201D\x2033\x2036" unicodeSingleQuotes = "\x2018\x2019" -prop_spacing = isOk spacing " \\\n # Comment" +prop_spacing1 = isOk spacing " \\\n # Comment" +prop_spacing2 = isOk spacing "# We can continue lines with \\" +prop_spacing3 = isWarning spacing " \\\n # --verbose=true \\" spacing = do - x <- many (many1 linewhitespace <|> try (string "\\\n" >> return "")) + x <- many (many1 linewhitespace <|> continuation) optional readComment return $ concat x + where + continuation = do + try (string "\\\n") + -- The line was continued. Warn if this next line is a comment with a trailing \ + whitespace <- many linewhitespace + optional $ do + x <- readComment + when ("\\" `isSuffixOf` x) $ + parseProblem ErrorC 1143 "This backslash is part of a comment and does not continue the line." + return whitespace spacing1 = do spacing <- spacing @@ -1040,13 +1052,9 @@ readComment = do readAnyComment prop_readAnyComment = isOk readAnyComment "# Comment" -prop_readAnyComment1 = not $ isOk readAnyComment "# Comment \\\n" readAnyComment = do char '#' - comment <- many $ noneOf "\\\r\n" - bs <- many $ oneOf "\\" - unless (null bs) (fail "Backslash in or directly after comment") - return comment + many $ noneOf "\r\n" prop_readNormalWord = isOk readNormalWord "'foo'\"bar\"{1..3}baz$(lol)" prop_readNormalWord2 = isOk readNormalWord "foo**(foo)!!!(@@(bar))" @@ -1409,6 +1417,8 @@ readNormalEscaped = called "escaped char" $ do do next <- quotable <|> oneOf "?*@!+[]{}.,~#" when (next == ' ') $ checkTrailingSpaces pos <|> return () + -- Check if this line is followed by a commented line with a trailing backslash + when (next == '\n') $ try . lookAhead $ void spacing return $ if next == '\n' then "" else [next] <|> do From 087865c68019597fc92e20f7e3276457018986d3 Mon Sep 17 00:00:00 2001 From: Matthias Diener Date: Sat, 20 Mar 2021 20:43:18 -0500 Subject: [PATCH 437/763] Clarify 'which' --- src/ShellCheck/Checks/Commands.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index b3e81a1..d7452e2 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -1056,7 +1056,7 @@ checkFindRedirections = CommandCheck (Basename "find") f prop_checkWhich = verify checkWhich "which '.+'" checkWhich = CommandCheck (Basename "which") $ - \t -> info (getId $ getCommandTokenOrThis t) 2230 "which is non-standard. Use builtin 'command -v' instead." + \t -> info (getId $ getCommandTokenOrThis t) 2230 "'which' is non-standard. Use builtin 'command -v' instead." prop_checkSudoRedirect1 = verify checkSudoRedirect "sudo echo 3 > /proc/file" prop_checkSudoRedirect2 = verify checkSudoRedirect "sudo cmd < input" From 5669eb22037980c5a6b74b0d420cb452990bcf88 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 11 Apr 2021 15:52:13 -0700 Subject: [PATCH 438/763] Make x-comparison warning default --- CHANGELOG.md | 2 +- src/ShellCheck/Analytics.hs | 10 ++-------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9eec957..f6d5a82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ - SC2264: Warn about wrapper functions that blatantly recurse - SC2265/SC2266: Warn when using & or | with test statements - SC2267: Warn when using xargs -i instead of -I -- Optional avoid-x-comparisons: Style warning SC2268 for `[ x$var = xval ]` +- SC2268: Warn about unnecessary x-comparisons like `[ x$var = xval ]` ### Fixed - SC1072/SC1073 now respond to disable annotations, though ignoring parse errors diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 9035e04..b53aee1 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -196,6 +196,7 @@ nodeChecks = [ ,checkAssignToSelf ,checkEqualsInCommand ,checkSecondArgIsComparison + ,checkComparisonWithLeadingX ] optionalChecks = map fst optionalTreeChecks @@ -243,13 +244,6 @@ optionalTreeChecks = [ cdPositive = "echo $VAR", cdNegative = "VAR=hello; echo $VAR" }, checkUnassignedReferences' True) - - ,(newCheckDescription { - cdName = "avoid-x-comparisons", - cdDescription = "Warn about 'x'-prefix in comparisons", - cdPositive = "[ \"x$var\" = xval ]", - cdNegative = "[ \"$var\" = val ]" - }, nodeChecksToTreeCheck [checkComparisonWithLeadingX]) ] optionalCheckMap :: Map.Map String (Parameters -> Token -> [TokenComment]) @@ -4006,7 +4000,7 @@ checkComparisonWithLeadingX params t = check lhs rhs _ -> return () where - msg = "Avoid outdated x-prefix in comparisons as it no longer serves a purpose." + msg = "Avoid x-prefix in comparisons as it no longer serves a purpose." check lhs rhs = sequence_ $ do l <- fixLeadingX lhs r <- fixLeadingX rhs From cff3e22911f25283ccef0a23bdfdfaafe3ad7c40 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 19 Apr 2021 14:31:13 -0700 Subject: [PATCH 439/763] Stable version v0.7.2 This release is dedicated to ethanol, for keeping COVID-19 off both our hands and our minds. --- CHANGELOG.md | 4 ++-- ShellCheck.cabal | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f6d5a82..4ff81cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## Git +## v0.7.2 - 2021-04-19 ### Added - `disable` directives can now be a range, e.g. `disable=SC3000-SC4000` - SC1143: Warn about line continuations in comments @@ -22,7 +22,7 @@ - POSIX/dash unsupported feature warnings now have individual SC3xxx codes - SC1090: A leading `$x/` or `$(x)/` is now treated as `./` when locating files - SC2154: Variables appearing in -z/-n tests are no longer considered unassigned -- SC2270-SC2285: Improved warnings about misused =, e.g. `${var}=42` +- SC2270-SC2285: Improved warnings about misused `=`, e.g. `${var}=42` ## v0.7.1 - 2020-04-04 diff --git a/ShellCheck.cabal b/ShellCheck.cabal index 2254c02..b474f47 100644 --- a/ShellCheck.cabal +++ b/ShellCheck.cabal @@ -1,5 +1,5 @@ Name: ShellCheck -Version: 0.7.1 +Version: 0.7.2 Synopsis: Shell script analysis tool License: GPL-3 License-file: LICENSE From aaa355472033b56ce56c10002ffd711feb5184ee Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 19 Apr 2021 16:40:25 -0700 Subject: [PATCH 440/763] Post-release CHANGELOG update --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ff81cf..af4b5a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +## Git +### Added + +### Fixed + +### Changed + + ## v0.7.2 - 2021-04-19 ### Added - `disable` directives can now be a range, e.g. `disable=SC3000-SC4000` From 2f26600653a26e890c850b64f8755bb5f5f302b7 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 19 Apr 2021 17:19:21 -0700 Subject: [PATCH 441/763] Update Cabal version for Hackage --- ShellCheck.cabal | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ShellCheck.cabal b/ShellCheck.cabal index b474f47..766b1d6 100644 --- a/ShellCheck.cabal +++ b/ShellCheck.cabal @@ -8,7 +8,7 @@ Author: Vidar Holen Maintainer: vidar@vidarholen.net Homepage: https://www.shellcheck.net/ Build-Type: Simple -Cabal-Version: >= 1.8 +Cabal-Version: >= 1.10 Bug-reports: https://github.com/koalaman/shellcheck/issues Description: The goals of ShellCheck are: @@ -83,6 +83,7 @@ library ShellCheck.Regex other-modules: Paths_ShellCheck + default-language: Haskell98 executable shellcheck if impl(ghc < 8.0) @@ -103,6 +104,7 @@ executable shellcheck QuickCheck >= 2.7.4, regex-tdfa, ShellCheck + default-language: Haskell98 main-is: shellcheck.hs test-suite test-shellcheck @@ -122,5 +124,6 @@ test-suite test-shellcheck QuickCheck >= 2.7.4, regex-tdfa, ShellCheck + default-language: Haskell98 main-is: test/shellcheck.hs From d47f3ff9866fde463cf318314387776608b799da Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 19 Apr 2021 22:46:35 -0700 Subject: [PATCH 442/763] Add wait between GitHub and Docker to allow replication --- .github/workflows/build.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 707b9a2..0acece2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -113,6 +113,10 @@ jobs: export TAGS="$(cat source/tags)" ./.github_deploy + - name: Waiting for GitHub to replicate uploaded releases + run: | + sleep 300 + - name: Upload to Docker Hub env: DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} From 9e60b3ea841bcaf48780bfcfc2e44aa6563a62de Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 22 Apr 2021 22:17:51 -0700 Subject: [PATCH 443/763] Fix haddock failures (fixes #2216) --- src/ShellCheck/Analytics.hs | 12 ++++++------ test/buildtest | 2 ++ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index b53aee1..b943cbc 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -4155,11 +4155,11 @@ checkEqualsInCommand params originalToken = _ | "===" `isPrefixOf` s -> borderMsg (getId originalToken) _ -> prefixMsg (getId cmd) - -- $var==42 + -- '$var==42' _ | "==" `isInfixOf` s -> badComparisonMsg (getId cmd) - -- ${foo[x]}=42 and $foo=42 + -- '${foo[x]}=42' and '$foo=42' [T_DollarBraced id braced l] | "=" `isPrefixOf` s -> do let variableStr = concat $ oversimplify l let variableReference = getBracedReference variableStr @@ -4172,22 +4172,22 @@ checkEqualsInCommand params originalToken = && "]" `isSuffixOf` variableModifier case () of - -- $foo=bar should already have caused a parse-time SC1066 + -- '$foo=bar' should already have caused a parse-time SC1066 -- _ | not braced && isPlain -> -- return () _ | variableStr == "" -> -- Don't try to fix ${}=foo genericMsg (getId cmd) - -- $#=42 or ${#var}=42 + -- '$#=42' or '${#var}=42' _ | "#" `isPrefixOf` variableStr -> genericMsg (getId cmd) - -- ${0}=42 + -- '${0}=42' _ | variableStr == "0" -> assign0Msg id $ fixWith [replaceToken id params "BASH_ARGV0"] - -- $2=2 + -- '$2=2' _ | isPositional -> positionalMsg id diff --git a/test/buildtest b/test/buildtest index 68bd048..1d194fc 100755 --- a/test/buildtest +++ b/test/buildtest @@ -29,6 +29,8 @@ cabal build || die "build failed" cabal test || die "test failed" +cabal haddock || + die "haddock failed" sc="$(find . -name shellcheck -type f -perm -111)" [ -x "$sc" ] || die "Can't find executable" From fe25a2b00e79a9be4371496541b5b15b9aa554bc Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 24 Apr 2021 17:08:10 -0700 Subject: [PATCH 444/763] Treat ${arr[*]} like $* for SC2048 --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 11 +++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index af4b5a0..e0295e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Fixed ### Changed +- SC2048: Warning about $\* now also applies to ${array[\*]} ## v0.7.2 - 2021-04-19 diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index b943cbc..97c0b27 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -820,10 +820,17 @@ checkShorthandIf _ _ = return () prop_checkDollarStar = verify checkDollarStar "for f in $*; do ..; done" prop_checkDollarStar2 = verifyNot checkDollarStar "a=$*" prop_checkDollarStar3 = verifyNot checkDollarStar "[[ $* = 'a b' ]]" +prop_checkDollarStar4 = verify checkDollarStar "for f in ${var[*]}; do ..; done" +prop_checkDollarStar5 = verify checkDollarStar "ls ${*//foo/bar}" +prop_checkDollarStar6 = verify checkDollarStar "ls ${var[*]%%.*}" checkDollarStar p t@(T_NormalWord _ [T_DollarBraced id _ l]) - | concat (oversimplify l) == "*" && - not (isStrictlyQuoteFree (parentMap p) t) = + | not (isStrictlyQuoteFree (parentMap p) t) = do + let str = concat (oversimplify l) + when ("*" `isPrefixOf` str) $ warn id 2048 "Use \"$@\" (with quotes) to prevent whitespace problems." + when ("[*]" `isPrefixOf` (getBracedModifier str)) $ + warn id 2048 "Use \"${array[@]}\" (with quotes) to prevent whitespace problems." + checkDollarStar _ _ = return () From 331e89be990547b6e21ad1b6e56065bcda1ba053 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 26 Apr 2021 10:33:36 -0700 Subject: [PATCH 445/763] Fix bad warning for ${#arr[*]}. Fixes #2218. --- src/ShellCheck/Analytics.hs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 97c0b27..e61c30a 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -823,12 +823,16 @@ prop_checkDollarStar3 = verifyNot checkDollarStar "[[ $* = 'a b' ]]" prop_checkDollarStar4 = verify checkDollarStar "for f in ${var[*]}; do ..; done" prop_checkDollarStar5 = verify checkDollarStar "ls ${*//foo/bar}" prop_checkDollarStar6 = verify checkDollarStar "ls ${var[*]%%.*}" +prop_checkDollarStar7 = verify checkDollarStar "ls ${*}" +prop_checkDollarStar8 = verifyNot checkDollarStar "ls ${#*}" +prop_checkDollarStar9 = verify checkDollarStar "ls ${arr[*]}" +prop_checkDollarStar10 = verifyNot checkDollarStar "ls ${#arr[*]}" checkDollarStar p t@(T_NormalWord _ [T_DollarBraced id _ l]) | not (isStrictlyQuoteFree (parentMap p) t) = do let str = concat (oversimplify l) when ("*" `isPrefixOf` str) $ warn id 2048 "Use \"$@\" (with quotes) to prevent whitespace problems." - when ("[*]" `isPrefixOf` (getBracedModifier str)) $ + when ("[*]" `isPrefixOf` (getBracedModifier str) && isVariableChar (headOrDefault '!' str)) $ warn id 2048 "Use \"${array[@]}\" (with quotes) to prevent whitespace problems." checkDollarStar _ _ = return () From ab369a35c91c61c6b915226e779f8611a5626f31 Mon Sep 17 00:00:00 2001 From: Jens Petersen Date: Tue, 18 May 2021 11:06:12 +0800 Subject: [PATCH 446/763] move readme to extra-doc-files and add changelog to releases --- ShellCheck.cabal | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ShellCheck.cabal b/ShellCheck.cabal index 766b1d6..9433f55 100644 --- a/ShellCheck.cabal +++ b/ShellCheck.cabal @@ -8,7 +8,7 @@ Author: Vidar Holen Maintainer: vidar@vidarholen.net Homepage: https://www.shellcheck.net/ Build-Type: Simple -Cabal-Version: >= 1.10 +Cabal-Version: 1.18 Bug-reports: https://github.com/koalaman/shellcheck/issues Description: The goals of ShellCheck are: @@ -22,9 +22,11 @@ Description: * To point out subtle caveats, corner cases and pitfalls, that may cause an advanced user's otherwise working script to fail under future circumstances. +Extra-Doc-Files: + README.md + CHANGELOG.md Extra-Source-Files: -- documentation - README.md shellcheck.1.md -- A script to build the man page using pandoc manpage @@ -126,4 +128,3 @@ test-suite test-shellcheck ShellCheck default-language: Haskell98 main-is: test/shellcheck.hs - From b61a7658d69c7f11be55f26f5d25186d4ac89cb6 Mon Sep 17 00:00:00 2001 From: Rebecca Cran Date: Mon, 24 May 2021 13:33:50 -0600 Subject: [PATCH 447/763] Fix typo in SC2006 message: "backticked" vs "backticks" --- src/ShellCheck/Analytics.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index e61c30a..da5f1f0 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1595,7 +1595,7 @@ prop_checkBackticks2 = verifyNot checkBackticks "echo $(foo)" prop_checkBackticks3 = verifyNot checkBackticks "echo `#inlined comment` foo" checkBackticks params (T_Backticked id list) | not (null list) = addComment $ - makeCommentWithFix StyleC id 2006 "Use $(...) notation instead of legacy backticked `...`." + makeCommentWithFix StyleC id 2006 "Use $(...) notation instead of legacy backticks `...`." (fixWith [replaceStart id params 1 "$(", replaceEnd id params 1 ")"]) checkBackticks _ _ = return () From 51009603037014d0536407f24e316acd7aa58580 Mon Sep 17 00:00:00 2001 From: Kamil Cukrowski Date: Wed, 26 May 2021 10:58:38 +0200 Subject: [PATCH 448/763] Add a comma to function characters Bash has very relaxed function name rules and a comma is also a valid character. This commit silences SC1036 check when a function name has a comma in its name. --- src/ShellCheck/Parser.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 3c16d5d..bf4fce4 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -66,7 +66,7 @@ doubleQuote = char '"' variableStart = upper <|> lower <|> oneOf "_" variableChars = upper <|> lower <|> digit <|> oneOf "_" -- Chars to allow in function names -functionChars = variableChars <|> oneOf ":+?-./^@" +functionChars = variableChars <|> oneOf ":+?-./^@," -- Chars to allow in functions using the 'function' keyword extendedFunctionChars = functionChars <|> oneOf "[]*=!" specialVariable = oneOf (concat specialVariables) From 163b2f12e2076a643e744774c8408370655bfc09 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 5 Jun 2021 18:16:22 -0700 Subject: [PATCH 449/763] Sanity check command names (fixes #2227) --- CHANGELOG.md | 2 ++ src/ShellCheck/Analytics.hs | 38 +++++++++++++++++++++++++++++++++++ src/ShellCheck/AnalyzerLib.hs | 4 ++++ 3 files changed, 44 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e0295e3..af762b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ ## Git ### Added +- SC2286-SC2288: Warn when command name ends in a symbol like `/.)'"` +- SC2289: Warn when command name contains tabs or linefeeds ### Fixed diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index e61c30a..7c44986 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -197,6 +197,7 @@ nodeChecks = [ ,checkEqualsInCommand ,checkSecondArgIsComparison ,checkComparisonWithLeadingX + ,checkCommandWithTrailingSymbol ] optionalChecks = map fst optionalTreeChecks @@ -4265,5 +4266,42 @@ checkSecondArgIsComparison _ t = T_NormalWord _ (x:_) -> getId x _ -> getId t + +prop_checkCommandWithTrailingSymbol1 = verify checkCommandWithTrailingSymbol "/" +prop_checkCommandWithTrailingSymbol2 = verify checkCommandWithTrailingSymbol "/foo/ bar/baz" +prop_checkCommandWithTrailingSymbol3 = verify checkCommandWithTrailingSymbol "/" +prop_checkCommandWithTrailingSymbol4 = verifyNot checkCommandWithTrailingSymbol "/*" +prop_checkCommandWithTrailingSymbol5 = verifyNot checkCommandWithTrailingSymbol "$foo/$bar" +prop_checkCommandWithTrailingSymbol6 = verify checkCommandWithTrailingSymbol "foo, bar" +prop_checkCommandWithTrailingSymbol7 = verifyNot checkCommandWithTrailingSymbol ". foo.sh" +prop_checkCommandWithTrailingSymbol8 = verifyNot checkCommandWithTrailingSymbol ": foo" +prop_checkCommandWithTrailingSymbol9 = verifyNot checkCommandWithTrailingSymbol "/usr/bin/python[23] file.py" + +checkCommandWithTrailingSymbol _ t = + case t of + T_SimpleCommand _ _ (cmd:_) -> + let str = fromJust $ getLiteralStringExt (\_ -> Just "x") cmd + last = lastOrDefault 'x' str + in + case str of + "." -> return () -- The . command + ":" -> return () -- The : command + " " -> return () -- Probably caught by SC1101 + "//" -> return () -- Probably caught by SC1127 + "" -> err (getId cmd) 2286 "This empty string is interpreted as a command name. Double check syntax (or use 'true' as a no-op)." + _ | last == '/' -> err (getId cmd) 2287 "This is interpreted as a command name ending with '/'. Double check syntax." + _ | last `elem` "\\.,([{<>}])#\"\'% " -> warn (getId cmd) 2288 ("This is interpreted as a command name ending with " ++ (format last) ++ ". Double check syntax.") + _ | '\t' `elem` str -> err (getId cmd) 2289 "This is interpreted as a command name containing a tab. Double check syntax." + _ | '\n' `elem` str -> err (getId cmd) 2289 "This is interpreted as a command name containing a linefeed. Double check syntax." + _ -> return () + _ -> return () + where + format x = + case x of + ' ' -> "space" + '\'' -> "apostrophe" + '\"' -> "doublequote" + x -> '\'' : x : "\'" + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 5b97e87..e4d76f8 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -877,6 +877,10 @@ getBracedModifier s = headOrDefault "" $ do headOrDefault _ (a:_) = a headOrDefault def _ = def +-- Get the last element or a default. Like `last` but safe. +lastOrDefault def [] = def +lastOrDefault _ list = last list + --- Get element n of a list, or Nothing. Like `!!` but safe. (!!!) list i = case drop i list of From 99f6554c9bd97d4815848b3aeba027ce196cbd41 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 18 Jul 2021 16:59:45 -0700 Subject: [PATCH 450/763] SC2181: Add '!' in suggestion as appropriate (fixes #2189) --- src/ShellCheck/Analytics.hs | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 3ae4d6a..88947a6 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -3118,24 +3118,29 @@ prop_checkReturnAgainstZero8 = verify checkReturnAgainstZero "(( $? ))" prop_checkReturnAgainstZero9 = verify checkReturnAgainstZero "(( ! $? ))" checkReturnAgainstZero _ token = case token of - TC_Binary id _ _ lhs rhs -> check lhs rhs - TA_Binary id _ lhs rhs -> check lhs rhs - TA_Unary id _ exp - | isExitCode exp -> message (getId exp) + TC_Binary id _ op lhs rhs -> check op lhs rhs + TA_Binary id op lhs rhs -> check op lhs rhs + TA_Unary id op exp + | isExitCode exp -> message (checksSuccessLhs op) (getId exp) TA_Sequence _ [exp] - | isExitCode exp -> message (getId exp) + | isExitCode exp -> message False (getId exp) _ -> return () where - check lhs rhs = + -- Is "$? op 0" trying to check if the command succeeded? + checksSuccessLhs op = not $ op `elem` ["-gt", "-ne", "!=", "!"] + -- Is "0 op $?" trying to check if the command succeeded? + checksSuccessRhs op = op `notElem` ["-ne", "!="] + check op lhs rhs = if isZero rhs && isExitCode lhs - then message (getId lhs) - else when (isZero lhs && isExitCode rhs) $ message (getId rhs) + then message (checksSuccessLhs op) (getId lhs) + else when (isZero lhs && isExitCode rhs) $ message (checksSuccessRhs op) (getId rhs) isZero t = getLiteralString t == Just "0" isExitCode t = case getWordParts t of [T_DollarBraced _ _ l] -> concat (oversimplify l) == "?" _ -> False - message id = style id 2181 "Check exit code directly with e.g. 'if mycmd;', not indirectly with $?." + message forSuccess id = style id 2181 $ + "Check exit code directly with e.g. 'if " ++ (if forSuccess then "" else "! ") ++ "mycmd;', not indirectly with $?." prop_checkRedirectedNowhere1 = verify checkRedirectedNowhere "> file" prop_checkRedirectedNowhere2 = verify checkRedirectedNowhere "> file | grep foo" From 9b077e28cbcf531f11176d4ac53688fc2ceb389b Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 21 Jul 2021 16:44:21 -0700 Subject: [PATCH 451/763] Add :/. to chars recognized for \alias suppression (fixes #2287) --- src/ShellCheck/Parser.hs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index bf4fce4..29dc4b0 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -2061,6 +2061,7 @@ prop_readSimpleCommand4 = isOk readSimpleCommand "typeset -a foo=(lol)" prop_readSimpleCommand5 = isOk readSimpleCommand "time if true; then echo foo; fi" prop_readSimpleCommand6 = isOk readSimpleCommand "time -p ( ls -l; )" prop_readSimpleCommand7 = isOk readSimpleCommand "\\ls" +prop_readSimpleCommand7b = isOk readSimpleCommand "\\:" prop_readSimpleCommand8 = isWarning readSimpleCommand "// Lol" prop_readSimpleCommand9 = isWarning readSimpleCommand "/* Lolbert */" prop_readSimpleCommand10 = isWarning readSimpleCommand "/**** Lolbert */" @@ -2321,7 +2322,7 @@ readCmdName = do -- Ignore alias suppression optional . try $ do char '\\' - lookAhead $ variableChars + lookAhead $ variableChars <|> oneOf ":." readCmdWord readCmdWord = do From 8be60028ef9ef52db907edb38f022802debe5b0b Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 22 Jul 2021 19:25:48 -0700 Subject: [PATCH 452/763] Don't warn when line starts with &> (fixes #2281) --- src/ShellCheck/Parser.hs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 29dc4b0..4a01775 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -1996,12 +1996,14 @@ readHereString = called "here string" $ do word <- readNormalWord return $ T_HereString id word +prop_readNewlineList1 = isOk readScript "&> /dev/null echo foo" readNewlineList = many1 ((linefeed <|> carriageReturn) `thenSkip` spacing) <* checkBadBreak where checkBadBreak = optional $ do pos <- getPosition try $ lookAhead (oneOf "|&") -- See if the next thing could be |, || or && + notFollowedBy2 (string "&>") -- Except &> or &>> which is valid parseProblemAt pos ErrorC 1133 "Unexpected start of line. If breaking lines, |/||/&& should be at the end of the previous one." readLineBreak = optional readNewlineList From 9eb63c97e6a5764b2fe0cb3e3e36cea5ade72a13 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 24 Jul 2021 13:07:05 -0700 Subject: [PATCH 453/763] Re-add warnings about 'declare var = value' (fixes #2279) --- CHANGELOG.md | 1 + src/ShellCheck/Checks/Commands.hs | 31 +++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index af762b6..92b65d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - SC2289: Warn when command name contains tabs or linefeeds ### Fixed +- SC2290: Warn about misused = in declare & co, which were not caught by SC2270+ ### Changed - SC2048: Warning about $\* now also applies to ${array[\*]} diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index d7452e2..0d8deae 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -95,6 +95,11 @@ commandChecks = [ ,checkSourceArgs ,checkChmodDashr ,checkXargsDashi + ,checkArgComparison "local" + ,checkArgComparison "declare" + ,checkArgComparison "export" + ,checkArgComparison "readonly" + ,checkArgComparison "typeset" ] optionalChecks = map fst optionalCommandChecks @@ -1143,5 +1148,31 @@ checkXargsDashi = CommandCheck (Basename "xargs") f return $ info (getId option) 2267 "GNU xargs -i is deprecated in favor of -I{}" parseOpts = getBsdOpts "0oprtxadR:S:J:L:l:n:P:s:e:E:i:I:" + +prop_checkArgComparison1 = verify (checkArgComparison "declare") "declare a = b" +prop_checkArgComparison2 = verify (checkArgComparison "declare") "declare a =b" +prop_checkArgComparison3 = verifyNot (checkArgComparison "declare") "declare a=b" +prop_checkArgComparison4 = verify (checkArgComparison "export") "export a +=b" +prop_checkArgComparison7 = verifyNot (checkArgComparison "declare") "declare -a +i foo" +-- This mirrors checkSecondArgIsComparison but for arguments to local/readonly/declare/export +checkArgComparison str = CommandCheck (Exactly str) wordsWithEqual + where + wordsWithEqual t = mapM_ check $ drop 1 $ arguments t + check arg = sequence_ $ do + str <- getLeadingUnquotedString arg + case str of + '=':_ -> + return $ err (headId arg) 2290 $ + "Remove spaces around = to assign." + '+':'=':_ -> + return $ err (headId arg) 2290 $ + "Remove spaces around += to append." + _ -> Nothing + + headId t = + case t of + T_NormalWord _ (x:_) -> getId x + _ -> getId t + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) From 0d58337cdd05d4d724ce1db4bcd6e1eb6c8721cf Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 25 Jul 2021 12:59:56 -0700 Subject: [PATCH 454/763] Don't warn about repeated range in [[ -v arr[xxx] ]] (fixes #2285) --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 13 ++++++++++++- src/ShellCheck/AnalyzerLib.hs | 6 +++--- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 92b65d7..62e516a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - SC2289: Warn when command name contains tabs or linefeeds ### Fixed +- SC2102 about repetitions in ranges no longer triggers on [[ -v arr[xx] ]] - SC2290: Warn about misused = in declare & co, which were not caught by SC2270+ ### Changed diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 88947a6..141529d 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2519,8 +2519,10 @@ prop_checkCharRangeGlob3 = verify checkCharRangeGlob "ls [10-15]" prop_checkCharRangeGlob4 = verifyNot checkCharRangeGlob "ls [a-zA-Z]" prop_checkCharRangeGlob5 = verifyNot checkCharRangeGlob "tr -d [a-zA-Z]" -- tr has 2060 prop_checkCharRangeGlob6 = verifyNot checkCharRangeGlob "[[ $x == [!!]* ]]" +prop_checkCharRangeGlob7 = verifyNot checkCharRangeGlob "[[ -v arr[keykey] ]]" +prop_checkCharRangeGlob8 = verifyNot checkCharRangeGlob "[[ arr[keykey] -gt 1 ]]" checkCharRangeGlob p t@(T_Glob id str) | - isCharClass str && not (isParamTo (parentMap p) "tr" t) = + isCharClass str && not (isParamTo (parentMap p) "tr" t) && not (isDereferenced t) = if ":" `isPrefixOf` contents && ":" `isSuffixOf` contents && contents /= ":" @@ -2537,6 +2539,15 @@ checkCharRangeGlob p t@(T_Glob id str) | '!':rest -> rest '^':rest -> rest x -> x + + -- Check if this is a dereferencing context like [[ -v array[operandhere] ]] + isDereferenced = fromMaybe False . msum . map isDereferencingOp . getPath (parentMap p) + isDereferencingOp t = + case t of + TC_Binary _ DoubleBracket str _ _ -> return $ isDereferencingBinaryOp str + TC_Unary _ _ str _ -> return $ str == "-v" + T_SimpleCommand {} -> return False + _ -> Nothing checkCharRangeGlob _ _ = return () diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index e4d76f8..dd6a604 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -737,7 +737,7 @@ getReferencedVariables parents t = TC_Unary id _ "-v" token -> getIfReference t token TC_Unary id _ "-R" token -> getIfReference t token TC_Binary id DoubleBracket op lhs rhs -> - if isDereferencing op + if isDereferencingBinaryOp op then concatMap (getIfReference t) [lhs, rhs] else [] @@ -771,12 +771,12 @@ getReferencedVariables parents t = when (isDigit h) $ fail "is a number" return (context, token, getBracedReference str) - isDereferencing = (`elem` ["-eq", "-ne", "-lt", "-le", "-gt", "-ge"]) - isArithmeticAssignment t = case getPath parents t of this: TA_Assignment _ "=" lhs _ :_ -> lhs == t _ -> False +isDereferencingBinaryOp = (`elem` ["-eq", "-ne", "-lt", "-le", "-gt", "-ge"]) + dataTypeFrom defaultType v = (case v of T_Array {} -> DataArray; _ -> defaultType) $ SourceFrom [v] From 364c33395e2f2d5500307f01989f70241c247d5a Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 25 Jul 2021 14:41:49 -0700 Subject: [PATCH 455/763] Don't print colors when $TERM is 'dumb' or unset (fixes #2260) --- CHANGELOG.md | 1 + src/ShellCheck/Formatter/Format.hs | 21 ++++++++++++--------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 62e516a..9e87af8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ ### Fixed - SC2102 about repetitions in ranges no longer triggers on [[ -v arr[xx] ]] - SC2290: Warn about misused = in declare & co, which were not caught by SC2270+ +- The flag --color=auto no longer outputs color when TERM is "dumb" or unset ### Changed - SC2048: Warning about $\* now also applies to ${array[\*]} diff --git a/src/ShellCheck/Formatter/Format.hs b/src/ShellCheck/Formatter/Format.hs index cb7dfe6..53b59a4 100644 --- a/src/ShellCheck/Formatter/Format.hs +++ b/src/ShellCheck/Formatter/Format.hs @@ -28,6 +28,7 @@ import Data.Array import Data.List import System.IO import System.Info +import System.Environment -- A formatter that carries along an arbitrary piece of data data Formatter = Formatter { @@ -68,12 +69,14 @@ makeNonVirtual comments contents = shouldOutputColor :: ColorOption -> IO Bool -shouldOutputColor colorOption = do - term <- hIsTerminalDevice stdout - let windows = "mingw" `isPrefixOf` os - let isUsableTty = term && not windows - let useColor = case colorOption of - ColorAlways -> True - ColorNever -> False - ColorAuto -> isUsableTty - return useColor +shouldOutputColor colorOption = + case colorOption of + ColorAlways -> return True + ColorNever -> return False + ColorAuto -> do + isTerminal <- hIsTerminalDevice stdout + term <- lookupEnv "TERM" + let windows = "mingw" `isPrefixOf` os + let dumbTerm = term `elem` [Just "dumb", Just "", Nothing] + let isUsableTty = isTerminal && not windows && not dumbTerm + return isUsableTty From 44471b73cc6090bbb944c1096b945d2da868d125 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 25 Jul 2021 17:31:13 -0700 Subject: [PATCH 456/763] Have SC2155 trigger on 'typeset' as well (fixes #2262) --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 26 ------------ src/ShellCheck/Checks/Commands.hs | 67 ++++++++++++++++++++++++++++--- 3 files changed, 63 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e87af8..b7a5fc4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Fixed - SC2102 about repetitions in ranges no longer triggers on [[ -v arr[xx] ]] +- SC2155 now recognizes `typeset` and local read-only `declare` statements - SC2290: Warn about misused = in declare & co, which were not caught by SC2270+ - The flag --color=auto no longer outputs color when TERM is "dumb" or unset diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 141529d..e5fd4cc 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -169,7 +169,6 @@ nodeChecks = [ ,checkTestArgumentSplitting ,checkConcatenatedDollarAt ,checkTildeInPath - ,checkMaskedReturns ,checkReadWithoutR ,checkLoopVariableReassignment ,checkTrailingBracket @@ -2970,31 +2969,6 @@ checkTestArgumentSplitting params t = err (getId token) 2255 "[ ] does not apply arithmetic evaluation. Evaluate with $((..)) for numbers, or use string comparator for strings." -prop_checkMaskedReturns1 = verify checkMaskedReturns "f() { local a=$(false); }" -prop_checkMaskedReturns2 = verify checkMaskedReturns "declare a=$(false)" -prop_checkMaskedReturns3 = verify checkMaskedReturns "declare a=\"`false`\"" -prop_checkMaskedReturns4 = verify checkMaskedReturns "readonly a=$(false)" -prop_checkMaskedReturns5 = verify checkMaskedReturns "readonly a=\"`false`\"" -prop_checkMaskedReturns6 = verifyNot checkMaskedReturns "declare a; a=$(false)" -prop_checkMaskedReturns7 = verifyNot checkMaskedReturns "f() { local -r a=$(false); }" -prop_checkMaskedReturns8 = verifyNot checkMaskedReturns "a=$(false); readonly a" -checkMaskedReturns _ t@(T_SimpleCommand id _ (cmd:rest)) = sequence_ $ do - name <- getCommandName t - guard $ name `elem` ["declare", "export", "readonly"] - || name == "local" && "r" `notElem` map snd (getAllFlags t) - return $ mapM_ checkArgs rest - where - checkArgs (T_Assignment id _ _ _ word) | any hasReturn $ getWordParts word = - warn id 2155 "Declare and assign separately to avoid masking return values." - checkArgs _ = return () - - hasReturn t = case t of - T_Backticked {} -> True - T_DollarExpansion {} -> True - _ -> False -checkMaskedReturns _ _ = return () - - prop_checkReadWithoutR1 = verify checkReadWithoutR "read -a foo" prop_checkReadWithoutR2 = verifyNot checkReadWithoutR "read -ar foo" prop_checkReadWithoutR3 = verifyNot checkReadWithoutR "read -t 0" diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 0d8deae..ec012c2 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -95,12 +95,12 @@ commandChecks = [ ,checkSourceArgs ,checkChmodDashr ,checkXargsDashi - ,checkArgComparison "local" - ,checkArgComparison "declare" - ,checkArgComparison "export" - ,checkArgComparison "readonly" - ,checkArgComparison "typeset" ] + ++ map checkArgComparison declaringCommands + ++ map checkMaskedReturns declaringCommands + +declaringCommands = ["local", "declare", "export", "readonly", "typeset"] + optionalChecks = map fst optionalCommandChecks optionalCommandChecks :: [(CheckDescription, CommandCheck)] @@ -1174,5 +1174,62 @@ checkArgComparison str = CommandCheck (Exactly str) wordsWithEqual T_NormalWord _ (x:_) -> getId x _ -> getId t + +prop_checkMaskedReturns1 = verify (checkMaskedReturns "local") "f() { local a=$(false); }" +prop_checkMaskedReturns2 = verify (checkMaskedReturns "declare") "declare a=$(false)" +prop_checkMaskedReturns3 = verify (checkMaskedReturns "declare") "declare a=\"`false`\"" +prop_checkMaskedReturns4 = verify (checkMaskedReturns "readonly") "readonly a=$(false)" +prop_checkMaskedReturns5 = verify (checkMaskedReturns "readonly") "readonly a=\"`false`\"" +prop_checkMaskedReturns6 = verifyNot (checkMaskedReturns "declare") "declare a; a=$(false)" +prop_checkMaskedReturns7 = verifyNot (checkMaskedReturns "local") "f() { local -r a=$(false); }" +prop_checkMaskedReturns8 = verifyNot (checkMaskedReturns "readonly") "a=$(false); readonly a" +prop_checkMaskedReturns9 = verify (checkMaskedReturns "typeset") "#!/bin/ksh\n f() { typeset -r x=$(false); }" +prop_checkMaskedReturns10 = verifyNot (checkMaskedReturns "typeset") "#!/bin/ksh\n function f { typeset -r x=$(false); }" +prop_checkMaskedReturns11 = verifyNot (checkMaskedReturns "typeset") "#!/bin/bash\n f() { typeset -r x=$(false); }" +prop_checkMaskedReturns12 = verify (checkMaskedReturns "typeset") "typeset -r x=$(false);" +prop_checkMaskedReturns13 = verify (checkMaskedReturns "typeset") "f() { typeset -g x=$(false); }" +prop_checkMaskedReturns14 = verify (checkMaskedReturns "declare") "declare x=${ false; }" +prop_checkMaskedReturns15 = verify (checkMaskedReturns "declare") "f() { declare x=$(false); }" +checkMaskedReturns str = CommandCheck (Exactly str) checkCmd + where + checkCmd t = do + path <- getPathM t + shell <- asks shellType + sequence_ $ do + name <- getCommandName t + + let flags = map snd (getAllFlags t) + let hasDashR = "r" `elem` flags + let hasDashG = "g" `elem` flags + let isInScopedFunction = any (isScopedFunction shell) path + + let isLocal = not hasDashG && isLocalInFunction name && isInScopedFunction + let isReadOnly = name == "readonly" || hasDashR + + -- Don't warn about local variables that are declared readonly, + -- because the workaround `local x; x=$(false); local -r x;` is annoying + guard . not $ isLocal && isReadOnly + + return $ mapM_ checkArgs $ arguments t + + checkArgs (T_Assignment id _ _ _ word) | any hasReturn $ getWordParts word = + warn id 2155 "Declare and assign separately to avoid masking return values." + checkArgs _ = return () + + isLocalInFunction = (`elem` ["local", "declare", "typeset"]) + isScopedFunction shell t = + case t of + T_BatsTest {} -> True + -- In ksh, only functions declared with 'function' have their own scope + T_Function _ (FunctionKeyword hasFunction) _ _ _ -> shell /= Ksh || hasFunction + _ -> False + + hasReturn t = case t of + T_Backticked {} -> True + T_DollarExpansion {} -> True + T_DollarBraceCommandExpansion {} -> True + _ -> False + + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) From 02e07625d178c5cda3b7c1e1ab90f3e68ed24837 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 25 Jul 2021 19:27:35 -0700 Subject: [PATCH 457/763] Warn about quoting in assignments to sh declaration utilities (fixes #1556) --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 14 ++++++++------ src/ShellCheck/AnalyzerLib.hs | 20 ++++++++++++++++---- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b7a5fc4..b43675a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ ### Changed - SC2048: Warning about $\* now also applies to ${array[\*]} +- Quote warnings are now emitted for declaration utilities in sh ## v0.7.2 - 2021-04-19 diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index e5fd4cc..6c4f04e 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -728,6 +728,7 @@ prop_checkUnquotedExpansions6 = verifyNot checkUnquotedExpansions "$(cmd)" prop_checkUnquotedExpansions7 = verifyNot checkUnquotedExpansions "cat << foo\n$(ls)\nfoo" prop_checkUnquotedExpansions8 = verifyNot checkUnquotedExpansions "set -- $(seq 1 4)" prop_checkUnquotedExpansions9 = verifyNot checkUnquotedExpansions "echo foo `# inline comment`" +prop_checkUnquotedExpansions10 = verify checkUnquotedExpansions "#!/bin/sh\nexport var=$(val)" checkUnquotedExpansions params = check where @@ -737,7 +738,7 @@ checkUnquotedExpansions params = check _ = return () tree = parentMap params examine t contents = - unless (null contents || shouldBeSplit t || isQuoteFree tree t || usedAsCommandName tree t) $ + unless (null contents || shouldBeSplit t || isQuoteFree (shellType params) tree t || usedAsCommandName tree t) $ warn (getId t) 2046 "Quote this to prevent word splitting." shouldBeSplit t = @@ -828,7 +829,7 @@ prop_checkDollarStar8 = verifyNot checkDollarStar "ls ${#*}" prop_checkDollarStar9 = verify checkDollarStar "ls ${arr[*]}" prop_checkDollarStar10 = verifyNot checkDollarStar "ls ${#arr[*]}" checkDollarStar p t@(T_NormalWord _ [T_DollarBraced id _ l]) - | not (isStrictlyQuoteFree (parentMap p) t) = do + | not (isStrictlyQuoteFree (shellType p) (parentMap p) t) = do let str = concat (oversimplify l) when ("*" `isPrefixOf` str) $ warn id 2048 "Use \"$@\" (with quotes) to prevent whitespace problems." @@ -849,7 +850,7 @@ prop_checkUnquotedDollarAt7 = verify checkUnquotedDollarAt "for f in ${var[@]}; prop_checkUnquotedDollarAt8 = verifyNot checkUnquotedDollarAt "echo \"${args[@]:+${args[@]}}\"" prop_checkUnquotedDollarAt9 = verifyNot checkUnquotedDollarAt "echo ${args[@]:+\"${args[@]}\"}" prop_checkUnquotedDollarAt10 = verifyNot checkUnquotedDollarAt "echo ${@+\"$@\"}" -checkUnquotedDollarAt p word@(T_NormalWord _ parts) | not $ isStrictlyQuoteFree (parentMap p) word = +checkUnquotedDollarAt p word@(T_NormalWord _ parts) | not $ isStrictlyQuoteFree (shellType p) (parentMap p) word = forM_ (find isArrayExpansion parts) $ \x -> unless (isQuotedAlternativeReference x) $ err (getId x) 2068 @@ -862,7 +863,7 @@ prop_checkConcatenatedDollarAt3 = verify checkConcatenatedDollarAt "echo $a$@" prop_checkConcatenatedDollarAt4 = verifyNot checkConcatenatedDollarAt "echo $@" prop_checkConcatenatedDollarAt5 = verifyNot checkConcatenatedDollarAt "echo \"${arr[@]}\"" checkConcatenatedDollarAt p word@T_NormalWord {} - | not $ isQuoteFree (parentMap p) word + | not $ isQuoteFree (shellType p) (parentMap p) word || null (drop 1 parts) = mapM_ for array where @@ -1891,6 +1892,7 @@ prop_checkSpacefulness40= verifyNotTree checkSpacefulness "a=$((x+1)); echo $a" prop_checkSpacefulness41= verifyNotTree checkSpacefulness "exec $1 --flags" prop_checkSpacefulness42= verifyNotTree checkSpacefulness "run $1 --flags" prop_checkSpacefulness43= verifyNotTree checkSpacefulness "$foo=42" +prop_checkSpacefulness44= verifyTree checkSpacefulness "#!/bin/dash\nexport var=$value" data SpaceStatus = SpaceSome | SpaceNone | SpaceEmpty deriving (Eq) instance Semigroup SpaceStatus where @@ -1972,7 +1974,7 @@ checkSpacefulness' onFind params t = isExpansion token && not (isArrayExpansion token) -- There's another warning for this && not (isCountingReference token) - && not (isQuoteFree parents token) + && not (isQuoteFree (shellType params) parents token) && not (isQuotedAlternativeReference token) && not (usedAsCommandName parents token) @@ -2090,7 +2092,7 @@ checkQuotesInLiterals params t = return $ case assignment of Just j | not (isParamTo parents "eval" expr) - && not (isQuoteFree parents expr) + && not (isQuoteFree (shellType params) parents expr) && not (squashesQuotes expr) -> [ makeComment WarningC j 2089 $ diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index dd6a604..dd957cd 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -287,14 +287,14 @@ isStrictlyQuoteFree = isQuoteFreeNode True isQuoteFree = isQuoteFreeNode False -isQuoteFreeNode strict tree t = +isQuoteFreeNode strict shell tree t = isQuoteFreeElement t || - headOrDefault False (mapMaybe isQuoteFreeContext (drop 1 $ getPath tree t)) + (fromMaybe False $ msum $ map isQuoteFreeContext $ drop 1 $ getPath tree t) where -- Is this node self-quoting in itself? isQuoteFreeElement t = case t of - T_Assignment {} -> True + T_Assignment {} -> assignmentIsQuoting t T_FdRedirect {} -> True _ -> False @@ -306,7 +306,7 @@ isQuoteFreeNode strict tree t = TC_Binary _ DoubleBracket _ _ _ -> return True TA_Sequence {} -> return True T_Arithmetic {} -> return True - T_Assignment {} -> return True + T_Assignment {} -> return $ assignmentIsQuoting t T_Redirecting {} -> return False T_DoubleQuoted _ _ -> return True T_DollarDoubleQuoted _ _ -> return True @@ -318,6 +318,18 @@ isQuoteFreeNode strict tree t = T_SelectIn {} -> return (not strict) _ -> Nothing + -- Check whether this assigment is self-quoting due to being a recognized + -- assignment passed to a Declaration Utility. This will soon be required + -- by POSIX: https://austingroupbugs.net/view.php?id=351 + assignmentIsQuoting t = shellParsesParamsAsAssignments || not (isAssignmentParamToCommand t) + shellParsesParamsAsAssignments = shell /= Sh + + -- Is this assignment a parameter to a command like export/typeset/etc? + isAssignmentParamToCommand (T_Assignment id _ _ _ _) = + case Map.lookup id tree of + Just (T_SimpleCommand _ _ (_:args)) -> id `elem` (map getId args) + _ -> False + -- Check if a token is a parameter to a certain command by name: -- Example: isParamTo (parentMap params) "sed" t isParamTo :: Map.Map Id Token -> String -> Token -> Bool From 4956b006ac46c0f03d74adbf51dbe573f5f3a859 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 25 Jul 2021 19:56:51 -0700 Subject: [PATCH 458/763] Fix broken test from previous commit --- src/ShellCheck/Analytics.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 6c4f04e..b65291a 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1892,7 +1892,7 @@ prop_checkSpacefulness40= verifyNotTree checkSpacefulness "a=$((x+1)); echo $a" prop_checkSpacefulness41= verifyNotTree checkSpacefulness "exec $1 --flags" prop_checkSpacefulness42= verifyNotTree checkSpacefulness "run $1 --flags" prop_checkSpacefulness43= verifyNotTree checkSpacefulness "$foo=42" -prop_checkSpacefulness44= verifyTree checkSpacefulness "#!/bin/dash\nexport var=$value" +prop_checkSpacefulness44= verifyTree checkSpacefulness "#!/bin/sh\nexport var=$value" data SpaceStatus = SpaceSome | SpaceNone | SpaceEmpty deriving (Eq) instance Semigroup SpaceStatus where From 754ab22d9485ad8d58b6a4943074066743f970ab Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 26 Jul 2021 18:29:55 -0700 Subject: [PATCH 459/763] Warn about unquoted blanks in echo (fixes #377) --- CHANGELOG.md | 1 + src/ShellCheck/Checks/Commands.hs | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b43675a..2d4e318 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ### Added - SC2286-SC2288: Warn when command name ends in a symbol like `/.)'"` - SC2289: Warn when command name contains tabs or linefeeds +- SC2291: Warn about repeated unquoted spaces between words in echo ### Fixed - SC2102 about repetitions in ranges no longer triggers on [[ -v arr[xx] ]] diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index ec012c2..1a08a7f 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -95,6 +95,7 @@ commandChecks = [ ,checkSourceArgs ,checkChmodDashr ,checkXargsDashi + ,checkUnquotedEchoSpaces ] ++ map checkArgComparison declaringCommands ++ map checkMaskedReturns declaringCommands @@ -1231,5 +1232,31 @@ checkMaskedReturns str = CommandCheck (Exactly str) checkCmd _ -> False +prop_checkUnquotedEchoSpaces1 = verify checkUnquotedEchoSpaces "echo foo bar" +prop_checkUnquotedEchoSpaces2 = verifyNot checkUnquotedEchoSpaces "echo foo" +prop_checkUnquotedEchoSpaces3 = verifyNot checkUnquotedEchoSpaces "echo foo bar" +prop_checkUnquotedEchoSpaces4 = verifyNot checkUnquotedEchoSpaces "echo 'foo bar'" +prop_checkUnquotedEchoSpaces5 = verifyNot checkUnquotedEchoSpaces "echo a > myfile.txt b" +prop_checkUnquotedEchoSpaces6 = verifyNot checkUnquotedEchoSpaces " echo foo\\\n bar" +checkUnquotedEchoSpaces = CommandCheck (Basename "echo") check + where + check t = do + let args = arguments t + m <- asks tokenPositions + redir <- getClosestCommandM t + sequence_ $ do + let positions = mapMaybe (\c -> Map.lookup (getId c) m) args + let pairs = zip positions (drop 1 positions) + (T_Redirecting _ redirTokens _) <- redir + let redirPositions = mapMaybe (\c -> fst <$> Map.lookup (getId c) m) redirTokens + guard $ any (hasSpacesBetween redirPositions) pairs + return $ info (getId t) 2291 "Quote repeated spaces to avoid them collapsing into one." + + hasSpacesBetween redirs ((a,b), (c,d)) = + posLine a == posLine d + && ((posColumn c) - (posColumn b)) >= 4 + && not (any (\x -> b < x && x < c) redirs) + + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) From c471e458224b449f73e9d2e9ad7fac670c262a08 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 26 Jul 2021 19:32:33 -0700 Subject: [PATCH 460/763] Allow printf/return/assignments after exec (fixes #2249) --- src/ShellCheck/Analytics.hs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index b65291a..f907ce3 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1705,6 +1705,7 @@ prop_checkSpuriousExec6 = verify checkSpuriousExec "exec foo > file; cmd" prop_checkSpuriousExec7 = verifyNot checkSpuriousExec "exec file; echo failed; exit 3" prop_checkSpuriousExec8 = verifyNot checkSpuriousExec "exec {origout}>&1- >tmp.log 2>&1; bar" prop_checkSpuriousExec9 = verify checkSpuriousExec "for file in rc.d/*; do exec \"$file\"; done" +prop_checkSpuriousExec10 = verifyNot checkSpuriousExec "exec file; r=$?; printf >&2 'failed\n'; return $r" checkSpuriousExec _ = doLists where doLists (T_Script _ _ cmds) = doList cmds False @@ -1720,7 +1721,8 @@ checkSpuriousExec _ = doLists stripCleanup = reverse . dropWhile cleanup . reverse cleanup (T_Pipeline _ _ [cmd]) = - isCommandMatch cmd (`elem` ["echo", "exit"]) + isCommandMatch cmd (`elem` ["echo", "exit", "printf", "return"]) + || isAssignment cmd cleanup _ = False doList = doList' . stripCleanup From fbc8d2cb2f8070f820c9337851bb97478e40e710 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 26 Jul 2021 20:48:47 -0700 Subject: [PATCH 461/763] Don't consider [ -n/-z/-v $var ] assignments for subshell modification (fixes #2217) --- src/ShellCheck/Analytics.hs | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index f907ce3..256c5b8 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1800,6 +1800,9 @@ prop_subshellAssignmentCheck17 = verifyNotTree subshellAssignmentCheck "foo=${ { prop_subshellAssignmentCheck18 = verifyTree subshellAssignmentCheck "( exec {n}>&2; ); echo $n" prop_subshellAssignmentCheck19 = verifyNotTree subshellAssignmentCheck "#!/bin/bash\nshopt -s lastpipe; echo a | read -r b; echo \"$b\"" prop_subshellAssignmentCheck20 = verifyTree subshellAssignmentCheck "@test 'foo' { a=1; }\n@test 'bar' { echo $a; }\n" +prop_subshellAssignmentCheck21 = verifyNotTree subshellAssignmentCheck "test1() { echo foo | if [[ $var ]]; then echo $var; fi; }; test2() { echo $var; }" +prop_subshellAssignmentCheck22 = verifyNotTree subshellAssignmentCheck "( [[ -n $foo || -z $bar ]] ); echo $foo $bar" +prop_subshellAssignmentCheck23 = verifyNotTree subshellAssignmentCheck "( export foo ); echo $foo" subshellAssignmentCheck params t = let flow = variableFlow params check = findSubshelled flow [("oops",[])] Map.empty @@ -1807,8 +1810,19 @@ subshellAssignmentCheck params t = findSubshelled [] _ _ = return () -findSubshelled (Assignment x@(_, _, str, _):rest) ((reason,scope):lol) deadVars = - findSubshelled rest ((reason, x:scope):lol) $ Map.insert str Alive deadVars +findSubshelled (Assignment x@(_, _, str, data_):rest) scopes@((reason,scope):restscope) deadVars = + if isTrueAssignment data_ + then findSubshelled rest ((reason, x:scope):restscope) $ Map.insert str Alive deadVars + else findSubshelled rest scopes deadVars + where + isTrueAssignment c = + case c of + DataString SourceChecked -> False + DataString SourceDeclaration -> False + DataArray SourceChecked -> False + DataArray SourceDeclaration -> False + _ -> True + findSubshelled (Reference (_, readToken, str):rest) scopes deadVars = do unless (shouldIgnore str) $ case Map.findWithDefault Alive str deadVars of Alive -> return () From fe81dc1c275f052856721bcac50f7dcf2e997c86 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 27 Jul 2021 18:53:30 -0700 Subject: [PATCH 462/763] Optionally suggest [[ over [ in Bash scripts (-o require-double-brackets) (fixes #887) --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 41 +++++++++++++++++++++++++++++++++++ src/ShellCheck/AnalyzerLib.hs | 3 ++- 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d4e318..272c694 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ - SC2286-SC2288: Warn when command name ends in a symbol like `/.)'"` - SC2289: Warn when command name contains tabs or linefeeds - SC2291: Warn about repeated unquoted spaces between words in echo +- SC2292: Suggest [[ over [ in Bash/Ksh scripts (optional) ### Fixed - SC2102 about repetitions in ranges no longer triggers on [[ -v arr[xx] ]] diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 256c5b8..f1c603e 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -244,6 +244,13 @@ optionalTreeChecks = [ cdPositive = "echo $VAR", cdNegative = "VAR=hello; echo $VAR" }, checkUnassignedReferences' True) + + ,(newCheckDescription { + cdName = "require-double-brackets", + cdDescription = "Require [[ and warn about [ in Bash/Ksh", + cdPositive = "[ -e /etc/issue ]", + cdNegative = "[[ -e /etc/issue ]]" + }, checkRequireDoubleBracket) ] optionalCheckMap :: Map.Map String (Parameters -> Token -> [TokenComment]) @@ -4311,5 +4318,39 @@ checkCommandWithTrailingSymbol _ t = '\"' -> "doublequote" x -> '\'' : x : "\'" + +prop_checkRequireDoubleBracket1 = verifyTree checkRequireDoubleBracket "[ -x foo ]" +prop_checkRequireDoubleBracket2 = verifyTree checkRequireDoubleBracket "[ foo -o bar ]" +prop_checkRequireDoubleBracket3 = verifyNotTree checkRequireDoubleBracket "#!/bin/sh\n[ -x foo ]" +prop_checkRequireDoubleBracket4 = verifyNotTree checkRequireDoubleBracket "[[ -x foo ]]" +checkRequireDoubleBracket params = + if isBashLike params + then nodeChecksToTreeCheck [check] params + else const [] + where + check _ t = case t of + T_Condition id SingleBracket _ -> + styleWithFix id 2292 "Prefer [[ ]] over [ ] for tests in Bash/Ksh." (fixFor t) + _ -> return () + + fixFor t = fixWith $ + if isSimple t + then + [ + replaceStart (getId t) params 0 "[", + replaceEnd (getId t) params 0 "]" + ] + else [] + + -- We don't tag operators like < and -o well enough to replace them, + -- so just handle the simple cases. + isSimple t = case t of + T_Condition _ _ s -> isSimple s + TC_Binary _ _ op _ _ -> not $ any (\x -> x `elem` op) "<>" + TC_Unary {} -> True + TC_Nullary {} -> True + _ -> False + + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index dd957cd..2e19a6e 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -178,7 +178,8 @@ makeCommentWithFix :: Severity -> Id -> Code -> String -> Fix -> TokenComment makeCommentWithFix severity id code str fix = let comment = makeComment severity id code str withFix = comment { - tcFix = Just fix + -- If fix is empty, pretend it wasn't there. + tcFix = if null (fixReplacements fix) then Nothing else Just fix } in force withFix From e33146d530986ded1c82e5bf8ab7dd3a36924e8e Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 29 Jul 2021 20:51:19 -0700 Subject: [PATCH 463/763] Avoid trigger SC2181 on composite $? checks (fixes #1167) --- CHANGELOG.md | 2 ++ src/ShellCheck/Analytics.hs | 61 ++++++++++++++++++++++++++++++++++--- 2 files changed, 59 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 272c694..f21857b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,11 +8,13 @@ ### Fixed - SC2102 about repetitions in ranges no longer triggers on [[ -v arr[xx] ]] - SC2155 now recognizes `typeset` and local read-only `declare` statements +- SC2181 now tries to avoid triggering for error handling functions - SC2290: Warn about misused = in declare & co, which were not caught by SC2270+ - The flag --color=auto no longer outputs color when TERM is "dumb" or unset ### Changed - SC2048: Warning about $\* now also applies to ${array[\*]} +- SC2181 now only triggers on single condition tests like `[ $? = 0 ]`. - Quote warnings are now emitted for declaration utilities in sh diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index f1c603e..d036c40 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -3126,20 +3126,71 @@ prop_checkReturnAgainstZero6 = verifyNot checkReturnAgainstZero "[[ $R -eq 0 ]]" prop_checkReturnAgainstZero7 = verify checkReturnAgainstZero "(( $? == 0 ))" prop_checkReturnAgainstZero8 = verify checkReturnAgainstZero "(( $? ))" prop_checkReturnAgainstZero9 = verify checkReturnAgainstZero "(( ! $? ))" -checkReturnAgainstZero _ token = +prop_checkReturnAgainstZero10 = verifyNot checkReturnAgainstZero "x=$(( $? > 0 ))" +prop_checkReturnAgainstZero11 = verify checkReturnAgainstZero "(( ! ! ! $? ))" +prop_checkReturnAgainstZero12 = verify checkReturnAgainstZero "[ ! $? -eq 0 ]" +prop_checkReturnAgainstZero13 = verifyNot checkReturnAgainstZero "(( ! $? && $? > 42))" +prop_checkReturnAgainstZero14 = verifyNot checkReturnAgainstZero "[[ -e foo || $? -eq 0 ]]" +prop_checkReturnAgainstZero15 = verifyNot checkReturnAgainstZero "(( $?, n=1 ))" +prop_checkReturnAgainstZero16 = verifyNot checkReturnAgainstZero "(( $? || $? == 4 ))" +prop_checkReturnAgainstZero17 = verifyNot checkReturnAgainstZero "(( $? + 0 ))" +prop_checkReturnAgainstZero18 = verifyNot checkReturnAgainstZero "f() { if [ $? -eq 0 ]; then :; fi; }" +prop_checkReturnAgainstZero19 = verifyNot checkReturnAgainstZero "f() ( [ $? -eq 0 ] || exit 42; )" +prop_checkReturnAgainstZero20 = verify checkReturnAgainstZero "f() { if :; then x; [ $? -eq 0 ] && exit; fi; }" +prop_checkReturnAgainstZero21 = verify checkReturnAgainstZero "(( ( $? ) ))" +prop_checkReturnAgainstZero22 = verify checkReturnAgainstZero "[[ ( $? -eq 0 ) ]]" +checkReturnAgainstZero params token = case token of TC_Binary id _ op lhs rhs -> check op lhs rhs - TA_Binary id op lhs rhs -> check op lhs rhs - TA_Unary id op exp + TA_Binary id op lhs rhs + | op `elem` [">", "<", ">=", "<=", "==", "!="] -> check op lhs rhs + TA_Unary id op@"!" exp | isExitCode exp -> message (checksSuccessLhs op) (getId exp) TA_Sequence _ [exp] | isExitCode exp -> message False (getId exp) _ -> return () where + -- We don't want to warn about composite expressions like + -- [[ $? -eq 0 || $? -eq 4 ]] since these can be annoying to rewrite. + isOnlyTestInCommand t = + case getPath (parentMap params) t of + _:(T_Condition {}):_ -> True + _:(T_Arithmetic {}):_ -> True + _:(TA_Sequence _ [_]):(T_Arithmetic {}):_ -> True + + -- Some negations and groupings are also fine + _:next@(TC_Unary _ _ "!" _):_ -> isOnlyTestInCommand next + _:next@(TA_Unary _ "!" _):_ -> isOnlyTestInCommand next + _:next@(TC_Group {}):_ -> isOnlyTestInCommand next + _:next@(TA_Sequence _ [_]):_ -> isOnlyTestInCommand next + _ -> False + + -- TODO: Do better $? tracking and filter on whether + -- the target command is in the same function + getFirstCommandInFunction = f + where + f t = case t of + T_Function _ _ _ _ x -> f x + T_BraceGroup _ (x:_) -> f x + T_Subshell _ (x:_) -> f x + T_Annotation _ _ x -> f x + T_AndIf _ x _ -> f x + T_OrIf _ x _ -> f x + T_Pipeline _ _ (x:_) -> f x + T_Redirecting _ _ (T_IfExpression _ (((x:_),_):_) _) -> f x + x -> x + + isFirstCommandInFunction = fromMaybe False $ do + let path = getPath (parentMap params) token + func <- listToMaybe $ filter isFunction path + cmd <- getClosestCommand (parentMap params) token + return $ getId cmd == getId (getFirstCommandInFunction func) + -- Is "$? op 0" trying to check if the command succeeded? checksSuccessLhs op = not $ op `elem` ["-gt", "-ne", "!=", "!"] -- Is "0 op $?" trying to check if the command succeeded? checksSuccessRhs op = op `notElem` ["-ne", "!="] + check op lhs rhs = if isZero rhs && isExitCode lhs then message (checksSuccessLhs op) (getId lhs) @@ -3149,9 +3200,11 @@ checkReturnAgainstZero _ token = case getWordParts t of [T_DollarBraced _ _ l] -> concat (oversimplify l) == "?" _ -> False - message forSuccess id = style id 2181 $ + + message forSuccess id = when (isOnlyTestInCommand token && not isFirstCommandInFunction) $ style id 2181 $ "Check exit code directly with e.g. 'if " ++ (if forSuccess then "" else "! ") ++ "mycmd;', not indirectly with $?." + prop_checkRedirectedNowhere1 = verify checkRedirectedNowhere "> file" prop_checkRedirectedNowhere2 = verify checkRedirectedNowhere "> file | grep foo" prop_checkRedirectedNowhere3 = verify checkRedirectedNowhere "grep foo | > bar" From a44f3edb1424e47fa9be790dbd3e72e2ff2904bd Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Fri, 30 Jul 2021 18:46:19 -0700 Subject: [PATCH 464/763] Warn about eval'ing arrays --- CHANGELOG.md | 1 + src/ShellCheck/AnalyzerLib.hs | 2 ++ src/ShellCheck/Checks/Commands.hs | 21 +++++++++++++++++++++ 3 files changed, 24 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f21857b..fe8d321 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - SC2289: Warn when command name contains tabs or linefeeds - SC2291: Warn about repeated unquoted spaces between words in echo - SC2292: Suggest [[ over [ in Bash/Ksh scripts (optional) +- SC2293/SC2294: Warn when calling `eval` with arrays ### Fixed - SC2102 about repetitions in ranges no longer triggers on [[ -v arr[xx] ]] diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 2e19a6e..beb4e5c 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -872,6 +872,8 @@ getBracedReference s = fromMaybe s $ prop_getBracedModifier1 = getBracedModifier "foo:bar:baz" == ":bar:baz" prop_getBracedModifier2 = getBracedModifier "!var:-foo" == ":-foo" prop_getBracedModifier3 = getBracedModifier "foo[bar]" == "[bar]" +prop_getBracedModifier4 = getBracedModifier "foo[@]@Q" == "[@]@Q" +prop_getBracedModifier5 = getBracedModifier "@@Q" == "@Q" getBracedModifier s = headOrDefault "" $ do let var = getBracedReference s a <- dropModifier s diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 1a08a7f..366438d 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -96,6 +96,7 @@ commandChecks = [ ,checkChmodDashr ,checkXargsDashi ,checkUnquotedEchoSpaces + ,checkEvalArray ] ++ map checkArgComparison declaringCommands ++ map checkMaskedReturns declaringCommands @@ -1258,5 +1259,25 @@ checkUnquotedEchoSpaces = CommandCheck (Basename "echo") check && not (any (\x -> b < x && x < c) redirs) +prop_checkEvalArray1 = verify checkEvalArray "eval $@" +prop_checkEvalArray2 = verify checkEvalArray "eval \"${args[@]}\"" +prop_checkEvalArray3 = verify checkEvalArray "eval \"${args[@]@Q}\"" +prop_checkEvalArray4 = verifyNot checkEvalArray "eval \"${args[*]@Q}\"" +prop_checkEvalArray5 = verifyNot checkEvalArray "eval \"$*\"" +checkEvalArray = CommandCheck (Exactly "eval") (mapM_ check . concatMap getWordParts . arguments) + where + check t = + when (isArrayExpansion t) $ + if isEscaped t + then style (getId t) 2293 "When eval'ing @Q-quoted words, use * rather than @ as the index." + else warn (getId t) 2294 "eval negates the benefit of arrays. Drop eval to preserve whitespace/symbols (or eval as string)." + + isEscaped q = + case q of + -- Match ${arr[@]@Q} and ${@@Q} and such + T_DollarBraced _ _ l -> 'Q' `elem` getBracedModifier (concat $ oversimplify l) + _ -> False + + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) From b939f86331f6343e1fb7f2fd84a26f84f608fd1b Mon Sep 17 00:00:00 2001 From: Yancharuk Alexander Date: Sat, 31 Jul 2021 06:24:20 +0300 Subject: [PATCH 465/763] Minor changes in README It's a recommended practice to use apt instead apt-get: >apt is a second command-line based front end provided by APT which overcomes some design mistakes of apt-get. https://debian-handbook.info/browse/stable/sect.apt-get.html Also added sudo for commands needed root privileges. --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index fe16fb2..551f7a8 100644 --- a/README.md +++ b/README.md @@ -143,7 +143,7 @@ On systems with Stack (installs to `~/.local/bin`): On Debian based distros: - apt-get install shellcheck + sudo apt install shellcheck On Arch Linux based distros: @@ -157,8 +157,8 @@ On Gentoo based distros: On EPEL based distros: - yum -y install epel-release - yum install ShellCheck + sudo yum -y install epel-release + sudo yum install ShellCheck On Fedora based distros: @@ -166,7 +166,7 @@ On Fedora based distros: On FreeBSD: - pkg install hs-ShellCheck + sudo pkg install hs-ShellCheck On macOS (OS X) with Homebrew: From 2f61b17518f51971eaee9cdb79631625bc3e6cae Mon Sep 17 00:00:00 2001 From: Yancharuk Alexander Date: Mon, 2 Aug 2021 19:09:24 +0300 Subject: [PATCH 466/763] Review fixes in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 551f7a8..dcfde31 100644 --- a/README.md +++ b/README.md @@ -166,7 +166,7 @@ On Fedora based distros: On FreeBSD: - sudo pkg install hs-ShellCheck + pkg install hs-ShellCheck On macOS (OS X) with Homebrew: From cf8066c07c063aca986d441583a4ee291dab74ae Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 3 Aug 2021 12:54:03 -0700 Subject: [PATCH 467/763] SC2295 Warn about unquoted variables in PE patterns (fixes #2290) --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 33 ++++++++++++++++++++++++++++++--- src/ShellCheck/AnalyzerLib.hs | 2 ++ 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fe8d321..2c99322 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ - SC2291: Warn about repeated unquoted spaces between words in echo - SC2292: Suggest [[ over [ in Bash/Ksh scripts (optional) - SC2293/SC2294: Warn when calling `eval` with arrays +- SC2295: Warn about "${x#$y}" treating $y as a pattern when not quoted ### Fixed - SC2102 about repetitions in ranges no longer triggers on [[ -v arr[xx] ]] diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index d036c40..a415ca2 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -197,6 +197,7 @@ nodeChecks = [ ,checkSecondArgIsComparison ,checkComparisonWithLeadingX ,checkCommandWithTrailingSymbol + ,checkUnquotedParameterExpansionPattern ] optionalChecks = map fst optionalTreeChecks @@ -388,7 +389,7 @@ replaceToken id params r = repInsertionPoint = InsertBefore } -surroundWidth id params s = fixWith [replaceStart id params 0 s, replaceEnd id params 0 s] +surroundWith id params s = fixWith [replaceStart id params 0 s, replaceEnd id params 0 s] fixWith fixes = newFix { fixReplacements = fixes } prop_checkEchoWc3 = verify checkEchoWc "n=$(echo $foo | wc -c)" @@ -1977,7 +1978,7 @@ quotesMayConflictWithSC2281 params t = (getId t) == (getId me) && (parentId == getId cmd) _ -> False -addDoubleQuotesAround params token = (surroundWidth (getId token) params "\"") +addDoubleQuotesAround params token = (surroundWith (getId token) params "\"") checkSpacefulness' :: (SpaceStatus -> Token -> String -> Writer [TokenComment] ()) -> Parameters -> Token -> [TokenComment] @@ -3274,7 +3275,7 @@ checkArrayAssignmentIndices params root = T_Literal id str <- parts let (before, after) = break ('=' ==) str guard $ all isDigit before && not (null after) - return $ warnWithFix id 2191 "The = here is literal. To assign by index, use ( [index]=value ) with no spaces. To keep as literal, quote it." (surroundWidth id params "\"") + return $ warnWithFix id 2191 "The = here is literal. To assign by index, use ( [index]=value ) with no spaces. To keep as literal, quote it." (surroundWith id params "\"") in if null literalEquals && isAssociative then warn (getId t) 2190 "Elements in associative arrays need index, e.g. array=( [index]=value ) ." @@ -4405,5 +4406,31 @@ checkRequireDoubleBracket params = _ -> False +prop_checkUnquotedParameterExpansionPattern1 = verify checkUnquotedParameterExpansionPattern "echo \"${var#$x}\"" +prop_checkUnquotedParameterExpansionPattern2 = verify checkUnquotedParameterExpansionPattern "echo \"${var%%$(x)}\"" +prop_checkUnquotedParameterExpansionPattern3 = verifyNot checkUnquotedParameterExpansionPattern "echo \"${var[#$x]}\"" +prop_checkUnquotedParameterExpansionPattern4 = verifyNot checkUnquotedParameterExpansionPattern "echo \"${var%\"$x\"}\"" + +checkUnquotedParameterExpansionPattern params x = + case x of + T_DollarBraced _ True word@(T_NormalWord _ (T_Literal _ s : rest@(_:_))) -> do + let modifier = getBracedModifier $ concat $ oversimplify word + when ("%" `isPrefixOf` modifier || "#" `isPrefixOf` modifier) $ + mapM_ check rest + _ -> return () + where + check t = + case t of + T_DollarBraced {} -> inform t + T_DollarExpansion {} -> inform t + T_Backticked {} -> inform t + _ -> return () + + inform t = + infoWithFix (getId t) 2295 + "Expansions inside ${..} need to be quoted separately, otherwise they match as patterns." $ + surroundWith (getId t) params "\"" + + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index beb4e5c..439b48f 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -167,6 +167,8 @@ errWithFix :: MonadWriter [TokenComment] m => Id -> Code -> String -> Fix -> m ( errWithFix = addCommentWithFix ErrorC warnWithFix :: MonadWriter [TokenComment] m => Id -> Code -> String -> Fix -> m () warnWithFix = addCommentWithFix WarningC +infoWithFix :: MonadWriter [TokenComment] m => Id -> Code -> String -> Fix -> m () +infoWithFix = addCommentWithFix InfoC styleWithFix :: MonadWriter [TokenComment] m => Id -> Code -> String -> Fix -> m () styleWithFix = addCommentWithFix StyleC From 378c9a2f2c396ac1323ebfcc599642a7e46ac133 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 3 Aug 2021 13:45:09 -0700 Subject: [PATCH 468/763] Switch build status badge from TravisCI to GitHub --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index dcfde31..bc0402c 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ -[![Build Status](https://travis-ci.org/koalaman/shellcheck.svg?branch=master)](https://travis-ci.org/koalaman/shellcheck) +[![Build Status](https://github.com/koalaman/shellcheck/actions/workflows/build.yml/badge.svg)](https://github.com/koalaman/shellcheck/actions/workflows/build.yml) + # ShellCheck - A shell script static analysis tool From 4dd762253f3aba35fe7a5070ab9395f062078637 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 3 Aug 2021 13:52:06 -0700 Subject: [PATCH 469/763] Remove defunct SonarQube plugin link (fixes #2292) --- README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/README.md b/README.md index bc0402c..a155b83 100644 --- a/README.md +++ b/README.md @@ -113,10 +113,6 @@ Services and platforms that have ShellCheck pre-installed and ready to use: * [CircleCI](https://circleci.com) via the [ShellCheck Orb](https://circleci.com/orbs/registry/orb/circleci/shellcheck) * [Github](https://github.com/features/actions) (only Linux) -Services and platforms with third party plugins: - -* [SonarQube](https://www.sonarqube.org/) through [sonar-shellcheck-plugin](https://github.com/emerald-squad/sonar-shellcheck-plugin) - Most other services, including [GitLab](https://about.gitlab.com/), let you install ShellCheck yourself, either through the system's package manager (see [Installing](#installing)), or by downloading and unpacking a [binary release](#installing-a-pre-compiled-binary). From e5745568e8b5ac18c2aa35a24aa6562c94380c74 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 8 Aug 2021 15:48:50 -0700 Subject: [PATCH 470/763] Extend warnings about spaces around = to 'let' --- src/ShellCheck/Checks/Commands.hs | 19 +++++++++++++++---- src/ShellCheck/Parser.hs | 2 +- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 366438d..c0cd9a3 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -101,7 +101,7 @@ commandChecks = [ ++ map checkArgComparison declaringCommands ++ map checkMaskedReturns declaringCommands -declaringCommands = ["local", "declare", "export", "readonly", "typeset"] +declaringCommands = ["local", "declare", "export", "readonly", "typeset", "let"] optionalChecks = map fst optionalCommandChecks @@ -1156,11 +1156,13 @@ prop_checkArgComparison2 = verify (checkArgComparison "declare") "declare a =b" prop_checkArgComparison3 = verifyNot (checkArgComparison "declare") "declare a=b" prop_checkArgComparison4 = verify (checkArgComparison "export") "export a +=b" prop_checkArgComparison7 = verifyNot (checkArgComparison "declare") "declare -a +i foo" +prop_checkArgComparison8 = verify (checkArgComparison "let") "let x = 0" -- This mirrors checkSecondArgIsComparison but for arguments to local/readonly/declare/export -checkArgComparison str = CommandCheck (Exactly str) wordsWithEqual +checkArgComparison cmd = CommandCheck (Exactly cmd) wordsWithEqual where - wordsWithEqual t = mapM_ check $ drop 1 $ arguments t - check arg = sequence_ $ do + wordsWithEqual t = mapM_ check $ arguments t + check arg = do + sequence_ $ do str <- getLeadingUnquotedString arg case str of '=':_ -> @@ -1171,6 +1173,15 @@ checkArgComparison str = CommandCheck (Exactly str) wordsWithEqual "Remove spaces around += to append." _ -> Nothing + -- 'let' is parsed as a sequence of arithmetic expansions, + -- so we want the additional warning for "x=" + when (cmd == "let") $ sequence_ $ do + token <- getTrailingUnquotedLiteral arg + str <- getLiteralString token + guard $ "=" `isSuffixOf` str + return $ err (getId token) 2290 $ + "Remove spaces around = to assign." + headId t = case t of T_NormalWord _ (x:_) -> getId x diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 4a01775..e559f62 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -2810,7 +2810,7 @@ readLetSuffix = many1 (readIoRedirect <|> try readLetExpression <|> readCmdWord) startPos <- getPosition expression <- readStringForParser readCmdWord let (unQuoted, newPos) = kludgeAwayQuotes expression startPos - subParse newPos readArithmeticContents unQuoted + subParse newPos (readArithmeticContents <* eof) unQuoted kludgeAwayQuotes :: String -> SourcePos -> (String, SourcePos) kludgeAwayQuotes s p = From fed4a048bccb7eccf30b5a18cab8e567114c8672 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Fri, 13 Aug 2021 23:11:20 -0700 Subject: [PATCH 471/763] Suppress SC2167 when name is "_" (fixes #2298) --- src/ShellCheck/Analytics.hs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index a415ca2..a45b2eb 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -3069,6 +3069,7 @@ checkUncheckedCdPushdPopd params root = prop_checkLoopVariableReassignment1 = verify checkLoopVariableReassignment "for i in *; do for i in *.bar; do true; done; done" prop_checkLoopVariableReassignment2 = verify checkLoopVariableReassignment "for i in *; do for((i=0; i<3; i++)); do true; done; done" prop_checkLoopVariableReassignment3 = verifyNot checkLoopVariableReassignment "for i in *; do for j in *.bar; do true; done; done" +prop_checkLoopVariableReassignment4 = verifyNot checkLoopVariableReassignment "for _ in *; do for _ in *.bar; do true; done; done" checkLoopVariableReassignment params token = sequence_ $ case token of T_ForIn {} -> check @@ -3077,6 +3078,7 @@ checkLoopVariableReassignment params token = where check = do str <- loopVariable token + guard $ str /= "_" next <- find (\x -> loopVariable x == Just str) path return $ do warn (getId token) 2165 "This nested loop overrides the index variable of its parent." From bb0a571a1e9ebf6aa2f92347f52103115fa494b6 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 16 Aug 2021 20:56:51 -0700 Subject: [PATCH 472/763] Improve warnings for bad parameter expansion (fixes #2297) --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 79 ++++++++++++++++++++++++++++------- src/ShellCheck/AnalyzerLib.hs | 15 ++++++- 3 files changed, 80 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c99322..ca29f28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - SC2292: Suggest [[ over [ in Bash/Ksh scripts (optional) - SC2293/SC2294: Warn when calling `eval` with arrays - SC2295: Warn about "${x#$y}" treating $y as a pattern when not quoted +- SC2296-SC2301: Improved warnings for bad parameter expansions ### Fixed - SC2102 about repetitions in ranges no longer triggers on [[ -v arr[xx] ]] diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index a45b2eb..d670977 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -136,7 +136,7 @@ nodeChecks = [ ,checkValidCondOps ,checkGlobbedRegex ,checkTestRedirects - ,checkIndirectExpansion + ,checkBadParameterSubstitution ,checkPS1Assignments ,checkBackticks ,checkInexplicablyUnquoted @@ -1608,29 +1608,79 @@ checkBackticks params (T_Backticked id list) | not (null list) = (fixWith [replaceStart id params 1 "$(", replaceEnd id params 1 ")"]) checkBackticks _ _ = return () -prop_checkIndirectExpansion1 = verify checkIndirectExpansion "${foo$n}" -prop_checkIndirectExpansion2 = verifyNot checkIndirectExpansion "${foo//$n/lol}" -prop_checkIndirectExpansion3 = verify checkIndirectExpansion "${$#}" -prop_checkIndirectExpansion4 = verify checkIndirectExpansion "${var${n}_$((i%2))}" -prop_checkIndirectExpansion5 = verifyNot checkIndirectExpansion "${bar}" -checkIndirectExpansion _ (T_DollarBraced i _ (T_NormalWord _ contents)) - | isIndirection contents = - err i 2082 "To expand via indirection, use arrays, ${!name} or (for sh only) eval." + +prop_checkBadParameterSubstitution1 = verify checkBadParameterSubstitution "${foo$n}" +prop_checkBadParameterSubstitution2 = verifyNot checkBadParameterSubstitution "${foo//$n/lol}" +prop_checkBadParameterSubstitution3 = verify checkBadParameterSubstitution "${$#}" +prop_checkBadParameterSubstitution4 = verify checkBadParameterSubstitution "${var${n}_$((i%2))}" +prop_checkBadParameterSubstitution5 = verifyNot checkBadParameterSubstitution "${bar}" +prop_checkBadParameterSubstitution6 = verify checkBadParameterSubstitution "${\"bar\"}" +prop_checkBadParameterSubstitution7 = verify checkBadParameterSubstitution "${{var}" +prop_checkBadParameterSubstitution8 = verify checkBadParameterSubstitution "${$(x)//x/y}" +prop_checkBadParameterSubstitution9 = verifyNot checkBadParameterSubstitution "$# ${#} $! ${!} ${!#} ${#!}" +prop_checkBadParameterSubstitution10 = verify checkBadParameterSubstitution "${'foo'}" +prop_checkBadParameterSubstitution11 = verify checkBadParameterSubstitution "${${x%.*}##*/}" + +checkBadParameterSubstitution _ t = + case t of + (T_DollarBraced i _ (T_NormalWord _ contents@(first:_))) -> + if isIndirection contents + then err i 2082 "To expand via indirection, use arrays, ${!name} or (for sh only) eval." + else checkFirst first + _ -> return () + where + isIndirection vars = let list = mapMaybe isIndirectionPart vars in not (null list) && and list + isIndirectionPart t = - case t of T_DollarExpansion _ _ -> Just True - T_Backticked _ _ -> Just True - T_DollarBraced _ _ _ -> Just True - T_DollarArithmetic _ _ -> Just True + case t of T_DollarExpansion {} -> Just True + T_Backticked {} -> Just True + T_DollarBraced {} -> Just True + T_DollarArithmetic {} -> Just True T_Literal _ s -> if all isVariableChar s then Nothing else Just False _ -> Just False -checkIndirectExpansion _ _ = return () + checkFirst t = + case t of + T_Literal id (c:_) -> + if isVariableChar c || isSpecialVariableChar c + then return () + else err id 2296 $ "Parameter expansions can't start with " ++ e4m [c] ++ ". Double check syntax." + + T_ParamSubSpecialChar {} -> return () + + T_DoubleQuoted id [T_Literal _ s] | isVariable s -> + err id 2297 "Double quotes must be outside ${}: ${\"invalid\"} vs \"${valid}\"." + + T_DollarBraced id braces _ | isUnmodifiedParameterExpansion t -> + err id 2298 $ + (if braces then "${${x}}" else "${$x}") + ++ " is invalid. For expansion, use ${x}. For indirection, use arrays, ${!x} or (for sh) eval." + + T_DollarBraced {} -> + err (getId t) 2299 "Parameter expansions can't be nested. Use temporary variables." + + _ | isCommandSubstitution t -> + err (getId t) 2300 "Parameter expansion can't be applied to command substitutions. Use temporary variables." + + _ -> err (getId t) 2301 $ "Parameter expansion starts with unexpected " ++ name t ++ ". Double check syntax." + + isVariable str = + case str of + [c] -> isVariableStartChar c || isSpecialVariableChar c || isDigit c + x -> isVariableName x + + name t = + case t of + T_SingleQuoted {} -> "quotes" + T_DoubleQuoted {} -> "quotes" + _ -> "syntax" + prop_checkInexplicablyUnquoted1 = verify checkInexplicablyUnquoted "echo 'var='value';'" prop_checkInexplicablyUnquoted2 = verifyNot checkInexplicablyUnquoted "'foo'*" @@ -4434,5 +4484,6 @@ checkUnquotedParameterExpansionPattern params x = surroundWith (getId t) params "\"" + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 439b48f..9ae0160 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -816,6 +816,7 @@ isConfusedGlobRegex _ = False isVariableStartChar x = x == '_' || isAsciiLower x || isAsciiUpper x isVariableChar x = isVariableStartChar x || isDigit x +isSpecialVariableChar = (`elem` "*@#?-$!") variableNameRegex = mkRegex "[_a-zA-Z][_a-zA-Z0-9]*" prop_isVariableName1 = isVariableName "_fo123" @@ -861,7 +862,7 @@ getBracedReference s = fromMaybe s $ let name = takeWhile isVariableChar s guard . not $ null name return name - getSpecial (c:_) | c `elem` "*@#?-$!" = return [c] + getSpecial (c:_) | isSpecialVariableChar c = return [c] getSpecial _ = fail "empty or not special" nameExpansion ('!':next:rest) = do -- e.g. ${!foo*bar*} @@ -955,5 +956,17 @@ isBashLike params = Dash -> False Sh -> False +-- Returns whether a token is a parameter expansion without any modifiers. +-- True for $var ${var} $1 $# +-- False for ${#var} ${var[x]} ${var:-0} +isUnmodifiedParameterExpansion t = + case t of + T_DollarBraced _ False _ -> True + T_DollarBraced _ _ list -> + let str = concat $ oversimplify list + in getBracedReference str == str + _ -> False + + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) From 8c0bf8d41f4aa18112acde9c9e594b66c93a2bdd Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 17 Aug 2021 12:50:40 -0700 Subject: [PATCH 473/763] Warn about looping over array values and using them as keys --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 84 +++++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca29f28..38b8e49 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - SC2293/SC2294: Warn when calling `eval` with arrays - SC2295: Warn about "${x#$y}" treating $y as a pattern when not quoted - SC2296-SC2301: Improved warnings for bad parameter expansions +- SC2302/SC2303: Warn about loops over array values when using them as keys ### Fixed - SC2102 about repetitions in ranges no longer triggers on [[ -v arr[xx] ]] diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index d670977..b768a0f 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -65,6 +65,7 @@ treeChecks = [ ,checkArrayAssignmentIndices ,checkUseBeforeDefinition ,checkAliasUsedInSameParsingUnit + ,checkArrayValueUsedAsIndex ] runAnalytics :: AnalysisSpec -> [TokenComment] @@ -4484,6 +4485,89 @@ checkUnquotedParameterExpansionPattern params x = surroundWith (getId t) params "\"" +prop_checkArrayValueUsedAsIndex1 = verifyTree checkArrayValueUsedAsIndex "for i in ${arr[@]}; do echo ${arr[i]}; done" +prop_checkArrayValueUsedAsIndex2 = verifyTree checkArrayValueUsedAsIndex "for i in ${arr[@]}; do echo ${arr[$i]}; done" +prop_checkArrayValueUsedAsIndex3 = verifyTree checkArrayValueUsedAsIndex "for i in ${arr[@]}; do echo $((arr[i])); done" +prop_checkArrayValueUsedAsIndex4 = verifyTree checkArrayValueUsedAsIndex "for i in ${arr1[@]} ${arr2[@]}; do echo ${arr1[$i]}; done" +prop_checkArrayValueUsedAsIndex5 = verifyTree checkArrayValueUsedAsIndex "for i in ${arr1[@]} ${arr2[@]}; do echo ${arr2[$i]}; done" +prop_checkArrayValueUsedAsIndex7 = verifyNotTree checkArrayValueUsedAsIndex "for i in ${arr[@]}; do echo ${arr[K]}; done" +prop_checkArrayValueUsedAsIndex8 = verifyNotTree checkArrayValueUsedAsIndex "for i in ${arr[@]}; do i=42; echo ${arr[i]}; done" +prop_checkArrayValueUsedAsIndex9 = verifyNotTree checkArrayValueUsedAsIndex "for i in ${arr[@]}; do echo ${arr2[i]}; done" + +checkArrayValueUsedAsIndex params _ = + doVariableFlowAnalysis read write Map.empty (variableFlow params) + where + write loop@T_ForIn {} _ name (DataString (SourceFrom words)) = do + modify $ Map.insert name (loop, mapMaybe f words) + return [] + where + f x = do + name <- getArrayName x + return (x, name) + + write _ _ name _ = do + modify $ Map.delete name + return [] + + read _ t name = do + varMap <- get + return $ fromMaybe [] $ do + (loop, arrays) <- Map.lookup name varMap + (arrayRef, arrayName) <- getArrayIfUsedAsIndex name t + -- Is this one of the 'for' arrays? + (loopWord, _) <- find ((==arrayName) . snd) arrays + -- Are we still in this loop? + guard $ getId loop `elem` map getId (getPath parents t) + return [ + makeComment WarningC (getId loopWord) 2302 "This loops over values. To loop over keys, use \"${!array[@]}\".", + makeComment WarningC (getId arrayRef) 2303 $ (e4m name) ++ " is an array value, not a key. Use directly or loop over keys instead." + ] + + parents = parentMap params + + getArrayName :: Token -> Maybe String + getArrayName t = do + [T_DollarBraced _ _ l] <- return $ getWordParts t + let str = concat $ oversimplify l + guard $ getBracedModifier str == "[@]" && not ("!" `isPrefixOf` str) + return $ getBracedReference str + + -- This is much uglier than it should be + getArrayIfUsedAsIndex :: String -> Token -> Maybe (Token, String) + getArrayIfUsedAsIndex name t = + case t of + T_DollarBraced _ _ list -> do + let ref = getBracedReference $ concat $ oversimplify list + guard $ ref == name + -- We found a $name. Look up the chain to see if it's ${arr[$name]} + list@T_NormalWord {} <- Map.lookup (getId t) parents + (T_DollarBraced _ _ parentList) <- Map.lookup (getId list) parents + (T_Literal _ head : index : T_Literal _ tail : _) <- return $ getWordParts parentList + let str = concat $ oversimplify list + let modifier = getBracedModifier str + guard $ getId index == getId t + guard $ "[${VAR}]" `isPrefixOf` modifier + return (t, getBracedReference str) + + T_NormalWord wordId list -> do + -- We found just name. Check if it's part of ${something[name]} + parent@(T_DollarBraced _ _ parentList) <- Map.lookup wordId parents + let str = concat $ oversimplify t + let modifier = getBracedModifier str + guard $ ("[" ++ name ++ "]") `isPrefixOf` modifier + return (parent, getBracedReference str) + + TA_Variable indexId ref [] -> do + -- We found arithmetic name. See if it's part of arithmetic arr[name] + guard $ ref == name + (TA_Sequence seqId [element]) <- Map.lookup indexId parents + guard $ getId element == indexId + parent@(TA_Variable arrayId arrayName [element]) <- Map.lookup seqId parents + guard $ getId element == seqId + return (parent, arrayName) + + _ -> Nothing + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) From c61fc7546e4660b117a89b2a3904c7317a1d694a Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 17 Aug 2021 14:14:05 -0700 Subject: [PATCH 474/763] Don't warn about variables guarded with :+ (fixes #2296) --- src/ShellCheck/Analytics.hs | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index b768a0f..9aff3ab 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2379,6 +2379,8 @@ prop_checkUnassignedReferences_minusNBraced = verifyNotTree checkUnassignedRefe prop_checkUnassignedReferences_minusZBraced = verifyNotTree checkUnassignedReferences "if [ -z \"${x}\" ]; then echo \"\"; fi" prop_checkUnassignedReferences_minusNDefault = verifyNotTree checkUnassignedReferences "if [ -n \"${x:-}\" ]; then echo $x; fi" prop_checkUnassignedReferences_minusZDefault = verifyNotTree checkUnassignedReferences "if [ -z \"${x:-}\" ]; then echo \"\"; fi" +prop_checkUnassignedReferences50 = verifyNotTree checkUnassignedReferences "echo ${foo:+bar}" +prop_checkUnassignedReferences51 = verifyNotTree checkUnassignedReferences "echo ${foo:+$foo}" checkUnassignedReferences = checkUnassignedReferences' False checkUnassignedReferences' includeGlobals params t = warnings @@ -2411,8 +2413,8 @@ checkUnassignedReferences' includeGlobals params t = warnings warningForGlobals var place = do match <- getBestMatch var - return $ warn (getId place) 2153 $ - "Possible misspelling: " ++ var ++ " may not be assigned, but " ++ match ++ " is." + return $ info (getId place) 2153 $ + "Possible misspelling: " ++ var ++ " may not be assigned. Did you mean " ++ match ++ "?" warningForLocals var place = return $ warn (getId place) 2154 $ @@ -2427,7 +2429,7 @@ checkUnassignedReferences' includeGlobals params t = warnings warningFor (var, place) = do guard $ isVariableName var - guard . not $ isInArray var place || isGuarded place + guard . not $ isException var place || isGuarded place (if includeGlobals || isLocal var then warningForLocals else warningForGlobals) var place @@ -2436,11 +2438,22 @@ checkUnassignedReferences' includeGlobals params t = warnings -- Due to parsing, foo=( [bar]=baz ) parses 'bar' as a reference even for assoc arrays. -- Similarly, ${foo[bar baz]} may not be referencing bar/baz. Just skip these. - isInArray var t = any isArray $ getPath (parentMap params) t + -- We can also have ${foo:+$foo} should be treated like [[ -n $foo ]] && echo $foo + isException var t = any shouldExclude $ getPath (parentMap params) t where - isArray T_Array {} = True - isArray (T_DollarBraced _ _ l) | var /= getBracedReference (concat $ oversimplify l) = True - isArray _ = False + shouldExclude t = + case t of + T_Array {} -> True + (T_DollarBraced _ _ l) -> + let str = concat $ oversimplify l + ref = getBracedReference str + mod = getBracedModifier str + in + -- Either we're used as an array index like ${arr[here]} + ref /= var || + -- or the reference is guarded by a parent, ${here:+foo$here} + "+" `isPrefixOf` mod || ":+" `isPrefixOf` mod + _ -> False isGuarded (T_DollarBraced _ _ v) = rest `matches` guardRegex From da7b28213ef65ef4a0d702831d1d358f37e4f114 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 17 Aug 2021 21:53:27 -0700 Subject: [PATCH 475/763] Recognize wait -p as assigning a variable (fixes #2179) --- src/ShellCheck/ASTLib.hs | 33 +++++++++++++++++++++++++++++++ src/ShellCheck/Analytics.hs | 2 ++ src/ShellCheck/AnalyzerLib.hs | 19 +++++++++--------- src/ShellCheck/Checks/Commands.hs | 12 +++++++++++ 4 files changed, 57 insertions(+), 9 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 019fc3c..7b5efdd 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -228,6 +228,39 @@ getOpts (gnu, arbitraryLongOpts) string longopts args = process args listToArgs = map (\x -> ("", (x, x))) + +-- Generic getOpts that doesn't rely on a format string, but may also be inaccurate. +-- This provides a best guess interpretation instead of failing when new options are added. +-- +-- "--" is treated as end of arguments +-- "--anything[=foo]" is treated as a long option without argument +-- "-any" is treated as -a -n -y, with the next arg as an option to -y unless it starts with - +-- anything else is an argument +getGenericOpts :: [Token] -> [(String, (Token, Token))] +getGenericOpts = process + where + process (token:rest) = + case getLiteralStringDef "\0" token of + "--" -> map (\c -> ("", (c,c))) rest + '-':'-':word -> (takeWhile (`notElem` "\0=") word, (token, token)) : process rest + '-':optString -> + let opts = takeWhile (/= '\0') optString + in + case rest of + next:_ | "-" `isPrefixOf` getLiteralStringDef "\0" next -> + map (\c -> ([c], (token, token))) opts ++ process rest + next:remainder -> + case reverse opts of + last:initial -> + map (\c -> ([c], (token, token))) (reverse initial) + ++ [([last], (token, next))] + ++ process remainder + [] -> process remainder + [] -> map (\c -> ([c], (token, token))) opts + _ -> ("", (token, token)) : process rest + process [] = [] + + -- Is this an expansion of multiple items of an array? isArrayExpansion (T_DollarBraced _ _ l) = let string = concat $ oversimplify l in diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 9aff3ab..e1e55fd 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1968,6 +1968,7 @@ prop_checkSpacefulness41= verifyNotTree checkSpacefulness "exec $1 --flags" prop_checkSpacefulness42= verifyNotTree checkSpacefulness "run $1 --flags" prop_checkSpacefulness43= verifyNotTree checkSpacefulness "$foo=42" prop_checkSpacefulness44= verifyTree checkSpacefulness "#!/bin/sh\nexport var=$value" +prop_checkSpacefulness45= verifyNotTree checkSpacefulness "wait -zzx -p foo; echo $foo" data SpaceStatus = SpaceSome | SpaceNone | SpaceEmpty deriving (Eq) instance Semigroup SpaceStatus where @@ -2381,6 +2382,7 @@ prop_checkUnassignedReferences_minusNDefault = verifyNotTree checkUnassignedRefe prop_checkUnassignedReferences_minusZDefault = verifyNotTree checkUnassignedReferences "if [ -z \"${x:-}\" ]; then echo \"\"; fi" prop_checkUnassignedReferences50 = verifyNotTree checkUnassignedReferences "echo ${foo:+bar}" prop_checkUnassignedReferences51 = verifyNotTree checkUnassignedReferences "echo ${foo:+$foo}" +prop_checkUnassignedReferences52 = verifyNotTree checkUnassignedReferences "wait -p pid; echo $pid" checkUnassignedReferences = checkUnassignedReferences' False checkUnassignedReferences' includeGlobals params t = warnings diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 9ae0160..633543a 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -617,6 +617,7 @@ getModifiedVariableCommand base@(T_SimpleCommand id cmdPrefix (T_NormalWord _ (T return (base, base, "@", DataString $ SourceFrom params) "printf" -> maybeToList $ getPrintfVariable rest + "wait" -> maybeToList $ getWaitVariable rest "mapfile" -> maybeToList $ getMapfileArray base rest "readarray" -> maybeToList $ getMapfileArray base rest @@ -674,15 +675,15 @@ getModifiedVariableCommand base@(T_SimpleCommand id cmdPrefix (T_NormalWord _ (T _ -> return (t:fromMaybe [] (getSetParams rest)) getSetParams [] = Nothing - getPrintfVariable list = f $ map (\x -> (x, getLiteralString x)) list - where - f ((_, Just "-v") : (t, Just var) : _) = return (base, t, varName, varType $ SourceFrom list) - where - (varName, varType) = case elemIndex '[' var of - Just i -> (take i var, DataArray) - Nothing -> (var, DataString) - f (_:rest) = f rest - f [] = fail "not found" + getPrintfVariable list = getFlagAssignedVariable "v" (SourceFrom list) $ getBsdOpts "v:" list + getWaitVariable list = getFlagAssignedVariable "p" SourceInteger $ return $ getGenericOpts list + + getFlagAssignedVariable str dataSource maybeFlags = do + flags <- maybeFlags + (_, (flag, value)) <- find ((== str) . fst) flags + variableName <- getLiteralStringExt (const $ return "!") value + let (baseName, index) = span (/= '[') variableName + return (base, value, baseName, (if null index then DataString else DataArray) dataSource) -- mapfile has some curious syntax allowing flags plus 0..n variable names -- where only the first non-option one is used if any. diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index c0cd9a3..6c7bdc5 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -138,18 +138,30 @@ prop_checkGetOptsS3 = checkGetOpts "-f -x" ["f", "x"] [] $ getOpts (True, True) prop_checkGetOptsS4 = checkGetOpts "-f -x" ["f"] [] $ getOpts (True, True) "f:" [] prop_checkGetOptsS5 = checkGetOpts "-fx" [] [] $ getOpts (True, True) "fx:" [] +prop_checkGenericOptsS1 = checkGetOpts "-f x" ["f"] [] $ return . getGenericOpts +prop_checkGenericOptsS2 = checkGetOpts "-abc x" ["a", "b", "c"] [] $ return . getGenericOpts +prop_checkGenericOptsS3 = checkGetOpts "-abc -x" ["a", "b", "c", "x"] [] $ return . getGenericOpts +prop_checkGenericOptsS4 = checkGetOpts "-x" ["x"] [] $ return . getGenericOpts + -- Long options prop_checkGetOptsL1 = checkGetOpts "--foo=bar baz" ["foo"] ["baz"] $ getOpts (True, False) "" [("foo", True)] prop_checkGetOptsL2 = checkGetOpts "--foo bar baz" ["foo"] ["baz"] $ getOpts (True, False) "" [("foo", True)] prop_checkGetOptsL3 = checkGetOpts "--foo baz" ["foo"] ["baz"] $ getOpts (True, True) "" [] prop_checkGetOptsL4 = checkGetOpts "--foo baz" [] [] $ getOpts (True, False) "" [] +prop_checkGenericOptsL1 = checkGetOpts "--foo=bar" ["foo"] [] $ return . getGenericOpts +prop_checkGenericOptsL2 = checkGetOpts "--foo bar" ["foo"] ["bar"] $ return . getGenericOpts +prop_checkGenericOptsL3 = checkGetOpts "-x --foo" ["x", "foo"] [] $ return . getGenericOpts + -- Know when to terminate prop_checkGetOptsT1 = checkGetOpts "-a x -b" ["a", "b"] ["x"] $ getOpts (True, True) "ab" [] prop_checkGetOptsT2 = checkGetOpts "-a x -b" ["a"] ["x","-b"] $ getOpts (False, True) "ab" [] prop_checkGetOptsT3 = checkGetOpts "-a -- -b" ["a"] ["-b"] $ getOpts (True, True) "ab" [] prop_checkGetOptsT4 = checkGetOpts "-a -- -b" ["a", "b"] [] $ getOpts (True, True) "a:b" [] +prop_checkGenericOptsT1 = checkGetOpts "-x -- -y" ["x"] ["-y"] $ return . getGenericOpts +prop_checkGenericOptsT2 = checkGetOpts "-xy --" ["x", "y"] [] $ return . getGenericOpts + buildCommandMap :: [CommandCheck] -> Map.Map CommandName (Token -> Analysis) buildCommandMap = foldl' addCheck Map.empty From 5b6fd60279a70539c27117f70e5386b984d3d9c3 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 22 Aug 2021 11:55:01 -0700 Subject: [PATCH 476/763] Improve warnings for expr (fixes #2033) --- CHANGELOG.md | 3 ++ src/ShellCheck/AnalyzerLib.hs | 2 +- src/ShellCheck/Checks/Commands.hs | 71 +++++++++++++++++++++++++++---- 3 files changed, 67 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 38b8e49..4b510ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ - SC2295: Warn about "${x#$y}" treating $y as a pattern when not quoted - SC2296-SC2301: Improved warnings for bad parameter expansions - SC2302/SC2303: Warn about loops over array values when using them as keys +- SC2304-SC2306: Warn about unquoted globs in expr arguments +- SC2307: Warn about insufficient number of arguments to expr +- SC2308: Suggest other approaches for non-standard expr extensions ### Fixed - SC2102 about repetitions in ranges no longer triggers on [[ -v arr[xx] ]] diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 633543a..662eff5 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -142,7 +142,7 @@ producesComments c s = do prRoot pr let spec = defaultSpec pr let params = makeParameters spec - return . not . null $ runChecker params c + return . not . null $ filterByAnnotation spec params $ runChecker params c makeComment :: Severity -> Id -> Code -> String -> TokenComment makeComment severity id code note = diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 6c7bdc5..76ac9a7 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -57,7 +57,7 @@ commandChecks :: [CommandCheck] commandChecks = [ checkTr ,checkFindNameGlob - ,checkNeedlessExpr + ,checkExpr ,checkGrepRe ,checkTrapQuotes ,checkReturn @@ -254,19 +254,74 @@ checkFindNameGlob = CommandCheck (Basename "find") (f . arguments) where acc b -prop_checkNeedlessExpr = verify checkNeedlessExpr "foo=$(expr 3 + 2)" -prop_checkNeedlessExpr2 = verify checkNeedlessExpr "foo=`echo \\`expr 3 + 2\\``" -prop_checkNeedlessExpr3 = verifyNot checkNeedlessExpr "foo=$(expr foo : regex)" -prop_checkNeedlessExpr4 = verifyNot checkNeedlessExpr "foo=$(expr foo \\< regex)" -checkNeedlessExpr = CommandCheck (Basename "expr") f where - f t = +prop_checkExpr = verify checkExpr "foo=$(expr 3 + 2)" +prop_checkExpr2 = verify checkExpr "foo=`echo \\`expr 3 + 2\\``" +prop_checkExpr3 = verifyNot checkExpr "foo=$(expr foo : regex)" +prop_checkExpr4 = verifyNot checkExpr "foo=$(expr foo \\< regex)" +prop_checkExpr5 = verify checkExpr "# shellcheck disable=SC2003\nexpr match foo bar" +prop_checkExpr6 = verify checkExpr "# shellcheck disable=SC2003\nexpr foo : fo*" +prop_checkExpr7 = verify checkExpr "# shellcheck disable=SC2003\nexpr 5 -3" +prop_checkExpr8 = verifyNot checkExpr "# shellcheck disable=SC2003\nexpr \"$@\"" +prop_checkExpr9 = verifyNot checkExpr "# shellcheck disable=SC2003\nexpr 5 $rest" +prop_checkExpr10 = verify checkExpr "# shellcheck disable=SC2003\nexpr length \"$var\"" +prop_checkExpr11 = verify checkExpr "# shellcheck disable=SC2003\nexpr foo > bar" +prop_checkExpr12 = verify checkExpr "# shellcheck disable=SC2003\nexpr 1 | 2" +prop_checkExpr13 = verify checkExpr "# shellcheck disable=SC2003\nexpr 1 * 2" +prop_checkExpr14 = verify checkExpr "# shellcheck disable=SC2003\nexpr \"$x\" >= \"$y\"" + +checkExpr = CommandCheck (Basename "expr") f where + f t = do when (all (`notElem` exceptions) (words $ arguments t)) $ style (getId $ getCommandTokenOrThis t) 2003 "expr is antiquated. Consider rewriting this using $((..)), ${} or [[ ]]." + + case arguments t of + [lhs, op, rhs] -> do + checkOp lhs + case getWordParts op of + [T_Glob _ "*"] -> + err (getId op) 2304 + "* must be escaped to multiply: \\*. Modern $((x * y)) avoids this issue." + [T_Literal _ ":"] | isGlob rhs -> + warn (getId rhs) 2305 + "Quote regex argument to expr to avoid it expanding as a glob." + _ -> return () + + [single] | not (willSplit single) -> + warn (getId single) 2307 + "'expr' expects 3+ arguments but sees 1. Make sure each operator/operand is a separate argument, and escape <>&|." + + [first, second] | + (fromMaybe "" $ getLiteralString first) /= "length" + && not (willSplit first || willSplit second) -> do + checkOp first + warn (getId t) 2307 + "'expr' expects 3+ arguments, but sees 2. Make sure each operator/operand is a separate argument, and escape <>&|." + + (first:rest) -> do + checkOp first + forM_ rest $ \t -> + -- We already find 95%+ of multiplication and regex earlier, so don't bother classifying this further. + when (isGlob t) $ warn (getId t) 2306 "Escape glob characters in arguments to expr to avoid pathname expansion." + + _ -> return () + -- These operators are hard to replicate in POSIX - exceptions = [ ":", "<", ">", "<=", ">=" ] + exceptions = [ ":", "<", ">", "<=", ">=", + -- We can offer better suggestions for these + "match", "length", "substr", "index"] words = mapMaybe getLiteralString + checkOp side = + case getLiteralString side of + Just "match" -> msg "'expr match' has unspecified results. Prefer 'expr str : regex'." + Just "length" -> msg "'expr length' has unspecified results. Prefer ${#var}." + Just "substr" -> msg "'expr substr' has unspecified results. Prefer 'cut' or ${var#???}." + Just "index" -> msg "'expr index' has unspecified results. Prefer x=${var%%[chars]*}; $((${#x}+1))." + _ -> return () + where + msg = info (getId side) 2308 + prop_checkGrepRe1 = verify checkGrepRe "cat foo | grep *.mp3" prop_checkGrepRe2 = verify checkGrepRe "grep -Ev cow*test *.mp3" From 7384cec3f6f4e29212c58e7d654e7853d4d173fe Mon Sep 17 00:00:00 2001 From: a1346054 <36859588+a1346054@users.noreply.github.com> Date: Wed, 25 Aug 2021 14:15:36 +0000 Subject: [PATCH 477/763] Fix redirect in LICENSE file The file was obtained from: https://www.gnu.org/licenses/gpl-3.0.txt --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 0507f1f..0df6056 100644 --- a/LICENSE +++ b/LICENSE @@ -681,4 +681,4 @@ into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Lesser General Public License instead of this License. But first, please read -. +. From 98c7934c46ec665bd1e2565cf3cb3f8b6c34e2ef Mon Sep 17 00:00:00 2001 From: a1346054 <36859588+a1346054@users.noreply.github.com> Date: Wed, 25 Aug 2021 16:17:56 +0000 Subject: [PATCH 478/763] Remove trailing whitespace --- .github_deploy | 1 - CHANGELOG.md | 6 +++--- snap/snapcraft.yaml | 4 ++-- striptests | 1 - 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.github_deploy b/.github_deploy index 3de0ac2..82c8ec5 100755 --- a/.github_deploy +++ b/.github_deploy @@ -26,4 +26,3 @@ do done gh release upload "$tag" "${files[@]}" --clobber || exit 1 done - diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b510ab..f479c95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -222,7 +222,7 @@ - SC2185: Suggest explicitly adding path for `find` - SC2184: Warn about unsetting globs (e.g. `unset foo[1]`) - SC2183: Warn about `printf` with more formatters than variables -- SC2182: Warn about ignored arguments with `printf` +- SC2182: Warn about ignored arguments with `printf` - SC2181: Suggest using command directly instead of `if [ $? -eq 0 ]` - SC1106: Warn when using `test` operators in `(( 1 -eq 2 ))` @@ -393,7 +393,7 @@ ### Added - SC2121: Warn about trying to `set` variables, e.g. `set var = value` - SC2120/SC2119: Warn when a function uses `$1..` if none are ever passed -- SC2117: Warn when using `su` in interactive mode, e.g. `su foo; whoami` +- SC2117: Warn when using `su` in interactive mode, e.g. `su foo; whoami` - SC2116: Detect useless use of echo, e.g. `for i in $(echo $var)` - SC2115/SC2114: Detect some catastrophic `rm -r "$empty/"` mistakes - SC1081: Warn when capitalizing keywords like `While` @@ -444,7 +444,7 @@ ### Removed - Suggestions about using parameter expansion over basename -- The `jsoncheck` binary. Use `shellcheck -f json` instead. +- The `jsoncheck` binary. Use `shellcheck -f json` instead. ## v0.2.0 - 2013-10-27 diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 2dc4831..e14b854 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -16,12 +16,12 @@ description: | advanced user's otherwise working script to fail under future circumstances. - By default ShellCheck can only check non-hidden files under /home, to make + By default ShellCheck can only check non-hidden files under /home, to make ShellCheck be able to check files under /media and /run/media you must connect it to the `removable-media` interface manually: # snap connect shellcheck:removable-media - + version: git base: core18 grade: stable diff --git a/striptests b/striptests index 8281f78..c68c16a 100755 --- a/striptests +++ b/striptests @@ -75,4 +75,3 @@ find . -name '.git' -prune -o -type f -name '*.hs' -print | do modify "$file" detestify done - From c85ce2cb06cce93cccea86b540528ae9e5dde153 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 26 Aug 2021 18:50:40 -0700 Subject: [PATCH 479/763] Add `rg` to list of commands ignored for SC2016 (fixes #2209) --- src/ShellCheck/Analytics.hs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index e1e55fd..97863dd 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1040,6 +1040,7 @@ checkSingleQuotedVariables params t@(T_SingleQuoted id s) = ,"dpkg-query" ,"jq" -- could also check that user provides --arg ,"rename" + ,"rg" ,"unset" ,"git filter-branch" ,"mumps -run %XCMD" From 81b7ee55980962a4631aef5bf98b3cc21822c5a4 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 26 Aug 2021 19:40:21 -0700 Subject: [PATCH 480/763] Don't warn about unused variables starting with _ (fixes #1498) --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b510ab..03c8395 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ - SC2048: Warning about $\* now also applies to ${array[\*]} - SC2181 now only triggers on single condition tests like `[ $? = 0 ]`. - Quote warnings are now emitted for declaration utilities in sh +- Leading `_` can now be used to suppress warnings about unused variables ## v0.7.2 - 2021-04-19 diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 97863dd..8e0589c 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2310,6 +2310,7 @@ prop_checkUnused44= verifyNotTree checkUnusedAssignments "DEFINE_string \"foo$ib prop_checkUnused45= verifyTree checkUnusedAssignments "readonly foo=bar" prop_checkUnused46= verifyTree checkUnusedAssignments "readonly foo=(bar)" prop_checkUnused47= verifyNotTree checkUnusedAssignments "a=1; alias hello='echo $a'" +prop_checkUnused48= verifyNotTree checkUnusedAssignments "_a=1" checkUnusedAssignments params t = execWriter (mapM_ warnFor unused) where flow = variableFlow params @@ -2326,8 +2327,9 @@ checkUnusedAssignments params t = execWriter (mapM_ warnFor unused) unused = Map.assocs $ Map.difference assignments references warnFor (name, token) = - warn (getId token) 2034 $ - name ++ " appears unused. Verify use (or export if used externally)." + unless ("_" `isPrefixOf` name) $ + warn (getId token) 2034 $ + name ++ " appears unused. Verify use (or export if used externally)." stripSuffix = takeWhile isVariableChar defaultMap = Map.fromList $ zip internalVariables $ repeat () From 081f7eba24f9d0e908a12cde540dcb181be7d679 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 26 Aug 2021 23:05:14 -0700 Subject: [PATCH 481/763] Fix parsing of [$var] (fixes #2309) --- src/ShellCheck/ASTLib.hs | 26 ++++++++++++++++++++++---- src/ShellCheck/Analytics.hs | 2 ++ src/ShellCheck/AnalyzerLib.hs | 29 ++++++++++++++++++----------- src/ShellCheck/Checks/Commands.hs | 3 +++ src/ShellCheck/Parser.hs | 13 +++++++++---- 5 files changed, 54 insertions(+), 19 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 7b5efdd..20e5be4 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -59,10 +59,28 @@ willSplit x = T_NormalWord _ l -> any willSplit l _ -> False -isGlob T_Extglob {} = True -isGlob T_Glob {} = True -isGlob (T_NormalWord _ l) = any isGlob l -isGlob _ = False +isGlob t = case t of + T_Extglob {} -> True + T_Glob {} -> True + T_NormalWord _ l -> any isGlob l || hasSplitRange l + _ -> False + where + -- foo[x${var}y] gets parsed as foo,[,x,$var,y], + -- so check if there's such an interval + hasSplitRange l = + let afterBracket = dropWhile (not . isHalfOpenRange) l + in any isClosingRange afterBracket + + isHalfOpenRange t = + case t of + T_Literal _ "[" -> True + _ -> False + + isClosingRange t = + case t of + T_Literal _ str -> ']' `elem` str + _ -> False + -- Is this shell word a constant? isConstant token = diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 8e0589c..f35fc0d 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2311,6 +2311,8 @@ prop_checkUnused45= verifyTree checkUnusedAssignments "readonly foo=bar" prop_checkUnused46= verifyTree checkUnusedAssignments "readonly foo=(bar)" prop_checkUnused47= verifyNotTree checkUnusedAssignments "a=1; alias hello='echo $a'" prop_checkUnused48= verifyNotTree checkUnusedAssignments "_a=1" +prop_checkUnused49= verifyNotTree checkUnusedAssignments "declare -A array; key=a; [[ -v array[$key] ]]" + checkUnusedAssignments params t = execWriter (mapM_ warnFor unused) where flow = variableFlow params diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 662eff5..5b389d7 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -497,14 +497,8 @@ getModifiedVariables t = -- Count [[ -v foo ]] as an "assignment". -- This is to prevent [ -v foo ] being unassigned or unused. - TC_Unary id _ "-v" token -> do - str <- fmap (takeWhile (/= '[')) $ -- Quoted index - flip getLiteralStringExt token $ \x -> - case x of - T_Glob _ s -> return s -- Unquoted index - _ -> [] - - guard . not . null $ str + TC_Unary id _ "-v" token -> maybeToList $ do + str <- getVariableForTestDashV token return (t, token, str, DataString SourceChecked) TC_Unary _ _ "-n" token -> markAsChecked t token @@ -724,6 +718,20 @@ getIndexReferences s = fromMaybe [] $ do where re = mkRegex "(\\[.*\\])" +-- Given a NormalWord like foo or foo[$bar], get foo. +-- Primarily used to get references for [[ -v foo[bar] ]] +getVariableForTestDashV :: Token -> Maybe String +getVariableForTestDashV t = do + str <- takeWhile ('[' /=) <$> getLiteralStringExt toStr t + guard $ isVariableName str + return str + where + -- foo[bar] gets parsed with [bar] as a glob, so undo that + toStr (T_Glob _ s) = return s + -- Turn foo[$x] into foo[\0] so that we can get the constant array name + -- in a non-constant expression (while filtering out foo$x[$y]) + toStr _ = return "\0" + prop_getOffsetReferences1 = getOffsetReferences ":bar" == ["bar"] prop_getOffsetReferences2 = getOffsetReferences ":bar:baz" == ["bar", "baz"] prop_getOffsetReferences3 = getOffsetReferences "[foo]:bar" == ["bar"] @@ -782,9 +790,8 @@ getReferencedVariables parents t = T_Glob _ s -> return s -- Also when parsed as globs _ -> [] - getIfReference context token = do - str@(h:_) <- getLiteralStringExt literalizer token - when (isDigit h) $ fail "is a number" + getIfReference context token = maybeToList $ do + str <- getVariableForTestDashV token return (context, token, getBracedReference str) isArithmeticAssignment t = case getPath parents t of diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 76ac9a7..d0ada73 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -845,6 +845,9 @@ checkAliasesExpandEarly = CommandCheck (Exactly "alias") (f . arguments) prop_checkUnsetGlobs1 = verify checkUnsetGlobs "unset foo[1]" prop_checkUnsetGlobs2 = verifyNot checkUnsetGlobs "unset foo" +prop_checkUnsetGlobs3 = verify checkUnsetGlobs "unset foo[$i]" +prop_checkUnsetGlobs4 = verify checkUnsetGlobs "unset foo[x${i}y]" +prop_checkUnsetGlobs5 = verifyNot checkUnsetGlobs "unset foo][" checkUnsetGlobs = CommandCheck (Exactly "unset") (mapM_ check . arguments) where check arg = diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index e559f62..45434a8 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -1372,6 +1372,8 @@ prop_readGlob5 = isOk readGlob "[^[:alpha:]1-9]" prop_readGlob6 = isOk readGlob "[\\|]" prop_readGlob7 = isOk readGlob "[^[]" prop_readGlob8 = isOk readGlob "[*?]" +prop_readGlob9 = isOk readGlob "[!]^]" +prop_readGlob10 = isOk readGlob "[]]" readGlob = readExtglob <|> readSimple <|> readClass <|> readGlobbyLiteral where readSimple = do @@ -1379,22 +1381,25 @@ readGlob = readExtglob <|> readSimple <|> readClass <|> readGlobbyLiteral c <- oneOf "*?" id <- endSpan start return $ T_Glob id [c] - -- Doesn't handle weird things like [^]a] and [$foo]. fixme? readClass = try $ do start <- startSpan char '[' - s <- many1 (predefined <|> readNormalLiteralPart "]" <|> globchars) + negation <- charToString (oneOf "!^") <|> return "" + leadingBracket <- charToString (oneOf "]") <|> return "" + s <- many (predefined <|> readNormalLiteralPart "]" <|> globchars) + guard $ not (null leadingBracket) || not (null s) char ']' id <- endSpan start - return $ T_Glob id $ "[" ++ concat s ++ "]" + return $ T_Glob id $ "[" ++ concat (negation:leadingBracket:s) ++ "]" where - globchars = fmap return . oneOf $ "!$[" ++ extglobStartChars + globchars = charToString $ oneOf $ "![" ++ extglobStartChars predefined = do try $ string "[:" s <- many1 letter string ":]" return $ "[:" ++ s ++ ":]" + charToString = fmap return readGlobbyLiteral = do start <- startSpan c <- extglobStart <|> char '[' From 9d64d78c320cfcd89e46d1de53f6c6d8e9487b4f Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 28 Aug 2021 21:19:45 -0700 Subject: [PATCH 482/763] Allow running this repo as a pre-commit hook --- .dockerignore | 9 +++------ .pre-commit-hooks.yaml | 6 ++++++ Dockerfile | 18 ++++++++++++++++++ test/distrotest | 30 ++++++++++++++++++++++++++++++ 4 files changed, 57 insertions(+), 6 deletions(-) create mode 100644 .pre-commit-hooks.yaml create mode 100644 Dockerfile diff --git a/.dockerignore b/.dockerignore index 39d8893..49f90c9 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,3 @@ -* -!LICENSE -!Setup.hs -!ShellCheck.cabal -!shellcheck.hs -!src +dist +dist-newstyle +.git diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml new file mode 100644 index 0000000..9c5b4bc --- /dev/null +++ b/.pre-commit-hooks.yaml @@ -0,0 +1,6 @@ +- id: shellcheck + name: shellcheck + description: Static analysis tool for shell scripts + types: [shell] + language: docker + entry: shellcheck diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..0f8e07c --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +# This file builds ShellCheck for pre-commit. +# +# It may also be useful for local development, but it is notably NOT +# used to build the official ShellCheck docker images. + +FROM ubuntu:20.04 +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y cabal-install +RUN cabal update + +# Install dependencies separately for more efficient iteration +COPY ShellCheck.cabal /build/ +RUN cd /build && cabal install --dependencies-only + +# Now build the rest +COPY . /build/ +RUN cd /build && cabal build shellcheck +RUN find /build -type f -name shellcheck -perm /111 -exec cp -f {} /bin \; +ENTRYPOINT ["/bin/shellcheck"] diff --git a/test/distrotest b/test/distrotest index 346a706..8aaa84d 100755 --- a/test/distrotest +++ b/test/distrotest @@ -1,6 +1,9 @@ #!/bin/bash # This script runs 'buildtest' on each of several distros # via Docker. + +# shellcheck disable=SC2016 + set -o pipefail exec 3>&1 4>&2 @@ -32,6 +35,33 @@ echo "Logging to $log" >&3 exec >> "$log" 2>&1 final=0 + +echo "Trying to build pre-commit docker image" +if ! docker build --tag precommit . +then + final=1 + echo "pre-commit image failed to build" +else + if printf '%s\n' '#!/bin/sh' 'echo $1' | docker run -i precommit - + then + final=1 + echo "pre-commit image succeeds with incorrect example" + fi + + if ! printf '%s\n' '#!/bin/sh' 'echo "$1"' | docker run -i precommit - + then + final=1 + echo "pre-commit image fails with correct example" + fi +fi + +if [[ $final -ne 0 ]] +then + echo >&3 "pre-commit image failure, see log" +else + echo >&3 "pre-commit image succeeded" +fi + while read -r distro setup do [[ "$distro" = "#"* || -z "$distro" ]] && continue From b0f05018c1acc96b76b5e4ac805ed86fc47dd3ae Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 29 Aug 2021 12:12:08 -0700 Subject: [PATCH 483/763] Revert "Allow running this repo as a pre-commit hook" This reverts commit 9d64d78c320cfcd89e46d1de53f6c6d8e9487b4f. --- .dockerignore | 9 ++++++--- .pre-commit-hooks.yaml | 6 ------ Dockerfile | 18 ------------------ test/distrotest | 30 ------------------------------ 4 files changed, 6 insertions(+), 57 deletions(-) delete mode 100644 .pre-commit-hooks.yaml delete mode 100644 Dockerfile diff --git a/.dockerignore b/.dockerignore index 49f90c9..39d8893 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,6 @@ -dist -dist-newstyle -.git +* +!LICENSE +!Setup.hs +!ShellCheck.cabal +!shellcheck.hs +!src diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml deleted file mode 100644 index 9c5b4bc..0000000 --- a/.pre-commit-hooks.yaml +++ /dev/null @@ -1,6 +0,0 @@ -- id: shellcheck - name: shellcheck - description: Static analysis tool for shell scripts - types: [shell] - language: docker - entry: shellcheck diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 0f8e07c..0000000 --- a/Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -# This file builds ShellCheck for pre-commit. -# -# It may also be useful for local development, but it is notably NOT -# used to build the official ShellCheck docker images. - -FROM ubuntu:20.04 -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y cabal-install -RUN cabal update - -# Install dependencies separately for more efficient iteration -COPY ShellCheck.cabal /build/ -RUN cd /build && cabal install --dependencies-only - -# Now build the rest -COPY . /build/ -RUN cd /build && cabal build shellcheck -RUN find /build -type f -name shellcheck -perm /111 -exec cp -f {} /bin \; -ENTRYPOINT ["/bin/shellcheck"] diff --git a/test/distrotest b/test/distrotest index 8aaa84d..346a706 100755 --- a/test/distrotest +++ b/test/distrotest @@ -1,9 +1,6 @@ #!/bin/bash # This script runs 'buildtest' on each of several distros # via Docker. - -# shellcheck disable=SC2016 - set -o pipefail exec 3>&1 4>&2 @@ -35,33 +32,6 @@ echo "Logging to $log" >&3 exec >> "$log" 2>&1 final=0 - -echo "Trying to build pre-commit docker image" -if ! docker build --tag precommit . -then - final=1 - echo "pre-commit image failed to build" -else - if printf '%s\n' '#!/bin/sh' 'echo $1' | docker run -i precommit - - then - final=1 - echo "pre-commit image succeeds with incorrect example" - fi - - if ! printf '%s\n' '#!/bin/sh' 'echo "$1"' | docker run -i precommit - - then - final=1 - echo "pre-commit image fails with correct example" - fi -fi - -if [[ $final -ne 0 ]] -then - echo >&3 "pre-commit image failure, see log" -else - echo >&3 "pre-commit image succeeded" -fi - while read -r distro setup do [[ "$distro" = "#"* || -z "$distro" ]] && continue From b5da99c6b0978880c948fbdf2b3b3cf449bfdb59 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 29 Aug 2021 12:28:58 -0700 Subject: [PATCH 484/763] Add pre-commit instructions --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index a155b83..8de2194 100644 --- a/README.md +++ b/README.md @@ -239,6 +239,20 @@ pandoc -s -f markdown-smart -t man shellcheck.1.md -o shellcheck.1 sudo mv shellcheck.1 /usr/share/man/man1 ``` +### pre-commit + +To run ShellCheck via [pre-commit](https://pre-commit.com/), add the Docker image to your `.pre-commit-config.yaml`: + +``` +- repo: local + hooks: + - id: shellcheck + name: shellcheck + language: docker_image + entry: koalaman/shellcheck:stable # or e.g. ":v0.7.2" + types: [shell] +``` + ### Travis CI Travis CI has now integrated ShellCheck by default, so you don't need to manually install it. From 10817533d6fa0a39804ac69e540084aa3037ceab Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 29 Aug 2021 17:08:09 -0700 Subject: [PATCH 485/763] Add shellcheck-precommit hook to README.md --- README.md | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 8de2194..d91f962 100644 --- a/README.md +++ b/README.md @@ -241,16 +241,15 @@ sudo mv shellcheck.1 /usr/share/man/man1 ### pre-commit -To run ShellCheck via [pre-commit](https://pre-commit.com/), add the Docker image to your `.pre-commit-config.yaml`: +To run ShellCheck via [pre-commit](https://pre-commit.com/), add the hook to your `.pre-commit-config.yaml`: ``` -- repo: local - hooks: - - id: shellcheck - name: shellcheck - language: docker_image - entry: koalaman/shellcheck:stable # or e.g. ":v0.7.2" - types: [shell] +repos: +- repo: https://github.com/koalaman/shellcheck-precommit + rev: v0.7.2 + hooks: + - id: shellcheck +# args: ["--severity=warning"] # Optionally only show errors and warnings ``` ### Travis CI From f5fd9c2fed8bf99e114ba109d9ea452935f5726e Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 30 Aug 2021 10:56:55 -0700 Subject: [PATCH 486/763] Improve warnings about unnecessary subshells (fixes #2169) --- src/ShellCheck/Analytics.hs | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index f35fc0d..aff5048 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -3742,21 +3742,27 @@ checkForLoopGlobVariables _ t = suggest t = info (getId t) 2231 "Quote expansions in this for loop glob to prevent wordsplitting, e.g. \"$dir\"/*.txt ." + prop_checkSubshelledTests1 = verify checkSubshelledTests "a && ( [ b ] || ! [ c ] )" prop_checkSubshelledTests2 = verify checkSubshelledTests "( [ a ] )" prop_checkSubshelledTests3 = verify checkSubshelledTests "( [ a ] && [ b ] || test c )" prop_checkSubshelledTests4 = verify checkSubshelledTests "( [ a ] && { [ b ] && [ c ]; } )" +prop_checkSubshelledTests5 = verifyNot checkSubshelledTests "( [[ ${var:=x} = y ]] )" +prop_checkSubshelledTests6 = verifyNot checkSubshelledTests "( [[ $((i++)) = 10 ]] )" +prop_checkSubshelledTests7 = verifyNot checkSubshelledTests "( [[ $((i+=1)) = 10 ]] )" +prop_checkSubshelledTests8 = verify checkSubshelledTests "# shellcheck disable=SC2234\nf() ( [[ x ]] )" + checkSubshelledTests params t = case t of - T_Subshell id list | all isTestStructure list -> + T_Subshell id list | all isTestStructure list && (not (hasAssignment t)) -> case () of -- Special case for if (test) and while (test) _ | isCompoundCondition (getPath (parentMap params) t) -> - style id 2233 "Remove superfluous (..) around condition." + style id 2233 "Remove superfluous (..) around condition to avoid subshell overhead." - -- Special case for ([ x ]) - _ | isSingleTest list -> - style id 2234 "Remove superfluous (..) around test command." + -- Special case for ([ x ]), except for func() ( [ x ] ) + _ | isSingleTest list && (not $ isFunctionBody (getPath (parentMap params) t)) -> + style id 2234 "Remove superfluous (..) around test command to avoid subshell overhead." -- General case for ([ x ] || [ y ] && etc) _ -> style id 2235 "Use { ..; } instead of (..) to avoid subshell overhead." @@ -3768,6 +3774,11 @@ checkSubshelledTests params t = [c] | isTestCommand c -> True _ -> False + isFunctionBody path = + case path of + (_:f:_) -> isFunction f + _ -> False + isTestStructure t = case t of T_Banged _ t -> isTestStructure t @@ -3798,6 +3809,19 @@ checkSubshelledTests params t = T_UntilExpression {} : _ -> True _ -> False + hasAssignment t = isNothing $ doAnalysis guardNotAssignment t + guardNotAssignment t = + case t of + TA_Assignment {} -> Nothing + TA_Unary _ s _ -> guard . not $ "++" `isInfixOf` s || "--" `isInfixOf` s + T_DollarBraced _ _ l -> + let str = concat $ oversimplify l + modifier = getBracedModifier str + in + guard . not $ "=" `isPrefixOf` modifier || ":=" `isPrefixOf` modifier + T_DollarBraceCommandExpansion {} -> Nothing + _ -> Just () + -- Skip any parent of a T_Subshell until we reach something interesting skippable t = case t of From 747bd8fd6af31f578bf8683ac9df9f29d5ca6d0b Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 30 Aug 2021 19:50:00 -0700 Subject: [PATCH 487/763] Warn about strings for numerical operators in [[ ]] (fixes #2312) --- src/ShellCheck/Analytics.hs | 46 +++++++++++++++++++++++++++++++------ 1 file changed, 39 insertions(+), 7 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index aff5048..cc554e0 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1109,6 +1109,12 @@ prop_checkNumberComparisons13 = verify checkNumberComparisons "[ $foo > $bar ]" prop_checkNumberComparisons14 = verifyNot checkNumberComparisons "[[ foo < bar ]]" prop_checkNumberComparisons15 = verifyNot checkNumberComparisons "[ $foo '>' $bar ]" prop_checkNumberComparisons16 = verify checkNumberComparisons "[ foo -eq 'y' ]" +prop_checkNumberComparisons17 = verify checkNumberComparisons "[[ 'foo' -eq 2 ]]" +prop_checkNumberComparisons18 = verify checkNumberComparisons "[[ foo -eq 2 ]]" +prop_checkNumberComparisons19 = verifyNot checkNumberComparisons "foo=1; [[ foo -eq 2 ]]" +prop_checkNumberComparisons20 = verify checkNumberComparisons "[[ 2 -eq / ]]" +prop_checkNumberComparisons21 = verify checkNumberComparisons "[[ foo -eq foo ]]" + checkNumberComparisons params (TC_Binary id typ op lhs rhs) = do if isNum lhs || isNum rhs then do @@ -1134,8 +1140,8 @@ checkNumberComparisons params (TC_Binary id typ op lhs rhs) = do when (op `elem` arithmeticBinaryTestOps) $ do mapM_ checkDecimals [lhs, rhs] - when (typ == SingleBracket) $ - checkStrings [lhs, rhs] + mapM_ checkString [lhs, rhs] + where hasStringComparison = shellType params /= Sh @@ -1148,19 +1154,45 @@ checkNumberComparisons params (TC_Binary id typ op lhs rhs) = do decimalError = "Decimals are not supported. " ++ "Either use integers only, or use bc or awk to compare." - checkStrings = - mapM_ stringError . find isNonNum + checkString t = + let + asString = getLiteralStringDef "\0" t + isVar = isVariableName asString + kind = if isVar then "a variable" else "an arithmetic expression" + fix = if isVar then "$var" else "$((expr))" + in + when (isNonNum t) $ + if typ == SingleBracket + then + err (getId t) 2170 $ + "Invalid number for " ++ op ++ ". Use " ++ seqv op ++ + " to compare as string (or use " ++ fix ++ + " to expand as " ++ kind ++ ")." + else + -- We should warn if any of the following holds: + -- The string is not a variable name + -- Any part of it is quoted + -- It's not a recognized variable name + when (not isVar || any isQuotes (getWordParts t) || asString `notElem` assignedVariables) $ + warn (getId t) 2309 $ + op ++ " treats this as " ++ kind ++ ". " ++ + "Use " ++ seqv op ++ " to compare as string (or expand explicitly with " ++ fix ++ ")." + + assignedVariables :: [String] + assignedVariables = mapMaybe f (variableFlow params) + where + f t = do + Assignment (_, _, name, _) <- return t + return name isNonNum t = not . all numChar $ onlyLiteralString t numChar x = isDigit x || x `elem` "+-. " - stringError t = err (getId t) 2170 $ - "Numerical " ++ op ++ " does not dereference in [..]. Expand or use string operator." - isNum t = case oversimplify t of [v] -> all isDigit v _ -> False + isFraction t = case oversimplify t of [v] -> isJust $ matchRegex floatRegex v From 40216487d643c5c8a636d3e8a1a20535c1c478bf Mon Sep 17 00:00:00 2001 From: Fabian Wolff Date: Thu, 2 Sep 2021 17:47:06 +0200 Subject: [PATCH 488/763] Do not suggest `grep -c` as a replacement for `grep -l/-L | wc -l` --- src/ShellCheck/Analytics.hs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index cc554e0..3d854aa 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -508,6 +508,8 @@ prop_checkPipePitfalls13 = verifyNot checkPipePitfalls "foo | grep bar | wc -c" prop_checkPipePitfalls14 = verifyNot checkPipePitfalls "foo | grep -o bar | wc -cmwL" prop_checkPipePitfalls15 = verifyNot checkPipePitfalls "foo | grep bar | wc -cmwL" prop_checkPipePitfalls16 = verifyNot checkPipePitfalls "foo | grep -r bar | wc -l" +prop_checkPipePitfalls17 = verifyNot checkPipePitfalls "foo | grep -l bar | wc -l" +prop_checkPipePitfalls18 = verifyNot checkPipePitfalls "foo | grep -L bar | wc -l" checkPipePitfalls _ (T_Pipeline id _ commands) = do for ["find", "xargs"] $ \(find:xargs:_) -> @@ -529,7 +531,9 @@ checkPipePitfalls _ (T_Pipeline id _ commands) = do let flagsGrep = maybe [] (map snd . getAllFlags) $ getCommand grep flagsWc = maybe [] (map snd . getAllFlags) $ getCommand wc in - unless (any (`elem` ["o", "only-matching", "r", "R", "recursive"]) flagsGrep || any (`elem` ["m", "chars", "w", "words", "c", "bytes", "L", "max-line-length"]) flagsWc || null flagsWc) $ + unless (any (`elem` ["l", "files-with-matches", "L", "files-without-matches", "o", "only-matching", "r", "R", "recursive"]) flagsGrep + || any (`elem` ["m", "chars", "w", "words", "c", "bytes", "L", "max-line-length"]) flagsWc + || null flagsWc) $ style (getId grep) 2126 "Consider using grep -c instead of grep|wc -l." didLs <- fmap or . sequence $ [ From dc9032fca553d13340087d5094323540c1359d1b Mon Sep 17 00:00:00 2001 From: Christian Nassif-Haynes Date: Mon, 23 Aug 2021 03:27:40 +1000 Subject: [PATCH 489/763] Show info about `set -e` suppression during function calls --- src/ShellCheck/Analytics.hs | 98 ++++++++++++++++++++++++++++++----- src/ShellCheck/AnalyzerLib.hs | 22 ++++++-- 2 files changed, 103 insertions(+), 17 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index e1e55fd..b524410 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -253,6 +253,13 @@ optionalTreeChecks = [ cdPositive = "[ -e /etc/issue ]", cdNegative = "[[ -e /etc/issue ]]" }, checkRequireDoubleBracket) + + ,(newCheckDescription { + cdName = "check-set-e-suppressed", + cdDescription = "Notify when set -e is suppressed during function invocation", + cdPositive = "set -e; func() { cp *.txt ~/backup; rm *.txt; }; func && echo ok", + cdNegative = "set -e; func() { cp *.txt ~/backup; rm *.txt; }; func; echo ok" + }, checkSetESuppressed) ] optionalCheckMap :: Map.Map String (Parameters -> Token -> [TokenComment]) @@ -393,6 +400,24 @@ replaceToken id params r = surroundWith id params s = fixWith [replaceStart id params 0 s, replaceEnd id params 0 s] fixWith fixes = newFix { fixReplacements = fixes } +analyse f t = execState (doAnalysis f t) [] + +-- Make a map from functions to definition IDs +functions t = Map.fromList $ analyse findFunctions t +findFunctions (T_Function id _ _ name _) + = modify ((name, id):) +findFunctions _ = return () + +-- Make a map from aliases to definition IDs +aliases t = Map.fromList $ analyse findAliases t +findAliases t@(T_SimpleCommand _ _ (_:args)) + | t `isUnqualifiedCommand` "alias" = mapM_ getAlias args +findAliases _ = return () +getAlias arg = + let string = onlyLiteralString arg + in when ('=' `elem` string) $ + modify ((takeWhile (/= '=') string, getId arg):) + prop_checkEchoWc3 = verify checkEchoWc "n=$(echo $foo | wc -c)" checkEchoWc _ (T_Pipeline id _ [a, b]) = when (acmd == ["echo", "${VAR}"]) $ @@ -2239,21 +2264,11 @@ checkFunctionsUsedExternally params t = findExecFlags = ["-exec", "-execdir", "-ok"] dropFlags = dropWhile (\x -> "-" `isPrefixOf` fst x) - -- Make a map from functions/aliases to definition IDs - analyse f t = execState (doAnalysis f t) [] - functions = Map.fromList $ analyse findFunctions t - findFunctions (T_Function id _ _ name _) = modify ((name, id):) - findFunctions t@(T_SimpleCommand id _ (_:args)) - | t `isUnqualifiedCommand` "alias" = mapM_ getAlias args - findFunctions _ = return () - getAlias arg = - let string = onlyLiteralString arg - in when ('=' `elem` string) $ - modify ((takeWhile (/= '=') string, getId arg):) + functionsAndAliases = Map.union (functions t) (aliases t) checkArg cmd (_, arg) = sequence_ $ do literalArg <- getUnquotedLiteral arg -- only consider unquoted literals - definitionId <- Map.lookup literalArg functions + definitionId <- Map.lookup literalArg functionsAndAliases return $ do warn (getId arg) 2033 "Shell functions can't be passed to external commands." @@ -4583,6 +4598,65 @@ checkArrayValueUsedAsIndex params _ = _ -> Nothing +prop_checkSetESuppressed1 = verifyTree checkSetESuppressed "set -e; f(){ :; }; x=$(f)" +prop_checkSetESuppressed2 = verifyNotTree checkSetESuppressed "f(){ :; }; x=$(f)" +prop_checkSetESuppressed3 = verifyNotTree checkSetESuppressed "set -e; f(){ :; }; x=$(set -e; f)" +prop_checkSetESuppressed4 = verifyTree checkSetESuppressed "set -e; f(){ :; }; baz=$(set -e; f) || :" +prop_checkSetESuppressed5 = verifyNotTree checkSetESuppressed "set -e; f(){ :; }; baz=$(echo \"\") || :" +prop_checkSetESuppressed6 = verifyTree checkSetESuppressed "set -e; f(){ :; }; f && echo" +prop_checkSetESuppressed7 = verifyTree checkSetESuppressed "set -e; f(){ :; }; f || echo" +prop_checkSetESuppressed8 = verifyNotTree checkSetESuppressed "set -e; f(){ :; }; echo && f" +prop_checkSetESuppressed9 = verifyNotTree checkSetESuppressed "set -e; f(){ :; }; echo || f" +prop_checkSetESuppressed10 = verifyTree checkSetESuppressed "set -e; f(){ :; }; ! f" +prop_checkSetESuppressed11 = verifyTree checkSetESuppressed "set -e; f(){ :; }; if f; then :; fi" +prop_checkSetESuppressed12 = verifyTree checkSetESuppressed "set -e; f(){ :; }; if set -e; f; then :; fi" +prop_checkSetESuppressed13 = verifyTree checkSetESuppressed "set -e; f(){ :; }; while f; do :; done" +prop_checkSetESuppressed14 = verifyTree checkSetESuppressed "set -e; f(){ :; }; while set -e; f; do :; done" +prop_checkSetESuppressed15 = verifyTree checkSetESuppressed "set -e; f(){ :; }; until f; do :; done" +prop_checkSetESuppressed16 = verifyTree checkSetESuppressed "set -e; f(){ :; }; until set -e; f; do :; done" +prop_checkSetESuppressed17 = verifyNotTree checkSetESuppressed "set -e; f(){ :; }; g(){ :; }; g f" +prop_checkSetESuppressed18 = verifyNotTree checkSetESuppressed "set -e; shopt -s inherit_errexit; f(){ :; }; x=$(f)" +checkSetESuppressed params t = + if hasSetE params then runNodeAnalysis checkNode params t else [] + where + checkNode _ (T_SimpleCommand _ _ (cmd:_)) = when (isFunction cmd) (checkCmd cmd) + checkNode _ _ = return () + + functions_ = functions t + + isFunction cmd = isJust $ do + literalArg <- getUnquotedLiteral cmd + Map.lookup literalArg functions_ + + checkCmd cmd = go $ getPath (parentMap params) cmd + where + go (child:parent:rest) = do + case parent of + T_Banged _ condition | child `isIn` [condition] -> informConditional "a ! condition" cmd + T_AndIf _ condition _ | child `isIn` [condition] -> informConditional "an && condition" cmd + T_OrIf _ condition _ | child `isIn` [condition] -> informConditional "an || condition" cmd + T_IfExpression _ condition _ | child `isIn` concatMap fst condition -> informConditional "an 'if' condition" cmd + T_UntilExpression _ condition _ | child `isIn` condition -> informConditional "an 'until' condition" cmd + T_WhileExpression _ condition _ | child `isIn` condition -> informConditional "a 'while' condition" cmd + T_DollarExpansion {} | not $ errExitEnabled parent -> informUninherited cmd + T_Backticked {} | not $ errExitEnabled parent -> informUninherited cmd + _ -> return () + go (parent:rest) + go _ = return () + + informConditional condType t = + info (getId t) 2310 ( + "This function is invoked in " ++ condType ++ " so set -e " ++ + "will be disabled. Invoke separately if failures should " ++ + "cause the script to exit.") + informUninherited t = + info (getId t) 2311 ( + "Bash implicitly disabled set -e for this function " ++ + "invocation because it's inside a command substitution. " ++ + "Add set -e; before it or enable inherit_errexit.") + errExitEnabled t = hasInheritErrexit params || containsSetE t + isIn t cmds = getId t `elem` map getId cmds + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 633543a..42d6f73 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -79,6 +79,8 @@ composeAnalyzers f g x = f x >> g x data Parameters = Parameters { -- Whether this script has the 'lastpipe' option set/default. hasLastpipe :: Bool, + -- Whether this script has the 'inherit_errexit' option set/default. + hasInheritErrexit :: Bool, -- Whether this script has 'set -e' anywhere. hasSetE :: Bool, -- A linear (bad) analysis of data flow @@ -196,7 +198,12 @@ makeParameters spec = Dash -> False Sh -> False Ksh -> True, - + hasInheritErrexit = + case shellType params of + Bash -> containsInheritErrexit root + Dash -> True + Sh -> True + Ksh -> False, shellTypeSpecified = isJust (asShellType spec) || isJust (asFallbackShell spec), parentMap = getParentTree root, variableFlow = getVariableFlow params root, @@ -219,18 +226,23 @@ containsSetE root = isNothing $ doAnalysis (guard . not . isSetE) root _ -> False re = mkRegex "[[:space:]]-[^-]*e" --- Does this script mention 'shopt -s lastpipe' anywhere? --- Also used as a hack. -containsLastpipe root = +containsShopt shopt root = isNothing $ doAnalysis (guard . not . isShoptLastPipe) root where isShoptLastPipe t = case t of T_SimpleCommand {} -> t `isUnqualifiedCommand` "shopt" && - ("lastpipe" `elem` oversimplify t) + (shopt `elem` oversimplify t) _ -> False +-- Does this script mention 'shopt -s inherit_errexit' anywhere? +containsInheritErrexit = containsShopt "inherit_errexit" + +-- Does this script mention 'shopt -s lastpipe' anywhere? +-- Also used as a hack. +containsLastpipe = containsShopt "lastpipe" + prop_determineShell0 = determineShellTest "#!/bin/sh" == Sh prop_determineShell1 = determineShellTest "#!/usr/bin/env ksh" == Ksh From 4e703e5c61c6366bfd486d728bc624025e344e68 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 15 Sep 2021 18:02:37 -0700 Subject: [PATCH 490/763] Allow specifying external-sources=true in shellcheckrc (fixes #1818) --- CHANGELOG.md | 4 +- shellcheck.1.md | 14 +++++ shellcheck.hs | 24 +++++---- src/ShellCheck/AST.hs | 1 + src/ShellCheck/Checker.hs | 75 ++++++++++++++++++++++++-- src/ShellCheck/Formatter/CheckStyle.hs | 2 +- src/ShellCheck/Formatter/Diff.hs | 2 +- src/ShellCheck/Formatter/GCC.hs | 2 +- src/ShellCheck/Formatter/JSON1.hs | 2 +- src/ShellCheck/Formatter/TTY.hs | 2 +- src/ShellCheck/Interface.hs | 14 +++-- src/ShellCheck/Parser.hs | 34 +++++++++--- 12 files changed, 145 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ae3cae..4dc25f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ -## Git +## Git (0.8.0) ### Added +- `external-sources=true` directive can be added to .shellcheckrc to make + shellcheck behave as if `-x` was specified. - SC2286-SC2288: Warn when command name ends in a symbol like `/.)'"` - SC2289: Warn when command name contains tabs or linefeeds - SC2291: Warn about repeated unquoted spaces between words in echo diff --git a/shellcheck.1.md b/shellcheck.1.md index d038df2..070c3a4 100644 --- a/shellcheck.1.md +++ b/shellcheck.1.md @@ -112,6 +112,9 @@ not warn at all, as `ksh` supports decimals in arithmetic contexts. line (plus `/dev/null`). This option allows following any file the script may `source`. + This option may also be enabled using `external-sources=true` in + `.shellcheckrc`. This flag takes precedence. + **FILES...** : One or more script files to check, or "-" for standard input. @@ -239,6 +242,14 @@ Valid keys are: : Enable an optional check by name, as listed with **--list-optional**. Only file-wide `enable` directives are considered. +**external-sources** +: Set to `true` in `.shellcheckrc` to always allow ShellCheck to open + arbitrary files from 'source' statements (the way most tools do). + + This option defaults to `false` only due to ShellCheck's origin as a + remote service for checking untrusted scripts. It can safely be enabled + for normal development. + **source** : Overrides the filename included by a `source`/`.` statement. This can be used to tell shellcheck where to look for a file whose name is determined @@ -270,6 +281,9 @@ Here is an example `.shellcheckrc`: source-path=SCRIPTDIR source-path=/mnt/chroot + # Allow opening any 'source'd file, even if not specified as input + external-sources=true + # Turn on warnings for unquoted variables with safe values enable=quote-safe-variables diff --git a/shellcheck.hs b/shellcheck.hs index d7e818d..bf70445 100644 --- a/shellcheck.hs +++ b/shellcheck.hs @@ -234,7 +234,7 @@ runFormatter sys format options files = do process :: FilePath -> IO Status process filename = do - input <- siReadFile sys filename + input <- siReadFile sys Nothing filename either (reportFailure filename) check input where check contents = do @@ -389,6 +389,7 @@ parseOption flag options = throwError SyntaxFailure return (Prelude.read num :: Integer) +ioInterface :: Options -> [FilePath] -> IO (SystemInterface IO) ioInterface options files = do inputs <- mapM normalize files cache <- newIORef emptyCache @@ -402,14 +403,14 @@ ioInterface options files = do emptyCache :: Map.Map FilePath String emptyCache = Map.empty - get cache inputs file = do + get cache inputs rcSuggestsExternal file = do map <- readIORef cache case Map.lookup file map of Just x -> return $ Right x - Nothing -> fetch cache inputs file + Nothing -> fetch cache inputs rcSuggestsExternal file - fetch cache inputs file = do - ok <- allowable inputs file + fetch cache inputs rcSuggestsExternal file = do + ok <- allowable rcSuggestsExternal inputs file if ok then (do (contents, shouldCache) <- inputFile file @@ -417,13 +418,16 @@ ioInterface options files = do modifyIORef cache $ Map.insert file contents return $ Right contents ) `catch` handler - else return $ Left (file ++ " was not specified as input (see shellcheck -x).") + else + if rcSuggestsExternal == Just False + then return $ Left (file ++ " was not specified as input, and external files were disabled via directive.") + else return $ Left (file ++ " was not specified as input (see shellcheck -x).") where handler :: IOException -> IO (Either ErrorMessage String) handler ex = return . Left $ show ex - allowable inputs x = - if externalSources options + allowable rcSuggestsExternal inputs x = + if fromMaybe (externalSources options) rcSuggestsExternal then return True else do path <- normalize x @@ -497,7 +501,7 @@ ioInterface options files = do b <- p x if b then pure (Just x) else acc - findSourceFile inputs sourcePathFlag currentScript sourcePathAnnotation original = + findSourceFile inputs sourcePathFlag currentScript rcSuggestsExternal sourcePathAnnotation original = if isAbsolute original then let (_, relative) = splitDrive original @@ -506,7 +510,7 @@ ioInterface options files = do find original original where find filename deflt = do - sources <- findM ((allowable inputs) `andM` doesFileExist) $ + sources <- findM ((allowable rcSuggestsExternal inputs) `andM` doesFileExist) $ (adjustPath filename):(map (( filename) . adjustPath) $ sourcePathFlag ++ sourcePathAnnotation) case sources of Nothing -> return deflt diff --git a/src/ShellCheck/AST.hs b/src/ShellCheck/AST.hs index 52b6e17..2cd2f6f 100644 --- a/src/ShellCheck/AST.hs +++ b/src/ShellCheck/AST.hs @@ -150,6 +150,7 @@ data Annotation = | SourceOverride String | ShellOverride String | SourcePath String + | ExternalSources Bool deriving (Show, Eq) data ConditionType = DoubleBracket | SingleBracket deriving (Show, Eq) diff --git a/src/ShellCheck/Checker.hs b/src/ShellCheck/Checker.hs index d81d664..514f97d 100644 --- a/src/ShellCheck/Checker.hs +++ b/src/ShellCheck/Checker.hs @@ -156,6 +156,11 @@ checkWithIncludesAndSourcePath includes mapper = getErrors siFindSource = mapper } +checkWithRcIncludesAndSourcePath rc includes mapper = getErrors + (mockRcFile rc $ mockedSystemInterface includes) { + siFindSource = mapper + } + prop_findsParseIssue = check "echo \"$12\"" == [1037] prop_commentDisablesParseIssue1 = @@ -384,7 +389,7 @@ prop_canEnableOptionalsWithRc = result == [2244] prop_sourcePathRedirectsName = result == [2086] where - f "dir/myscript" _ "lib" = return "foo/lib" + f "dir/myscript" _ _ "lib" = return "foo/lib" result = checkWithIncludesAndSourcePath [("foo/lib", "echo $1")] f emptyCheckSpec { csScript = "#!/bin/bash\nsource lib", csFilename = "dir/myscript", @@ -393,7 +398,7 @@ prop_sourcePathRedirectsName = result == [2086] prop_sourcePathAddsAnnotation = result == [2086] where - f "dir/myscript" ["mypath"] "lib" = return "foo/lib" + f "dir/myscript" _ ["mypath"] "lib" = return "foo/lib" result = checkWithIncludesAndSourcePath [("foo/lib", "echo $1")] f emptyCheckSpec { csScript = "#!/bin/bash\n# shellcheck source-path=mypath\nsource lib", csFilename = "dir/myscript", @@ -402,13 +407,75 @@ prop_sourcePathAddsAnnotation = result == [2086] prop_sourcePathRedirectsDirective = result == [2086] where - f "dir/myscript" _ "lib" = return "foo/lib" - f _ _ _ = return "/dev/null" + f "dir/myscript" _ _ "lib" = return "foo/lib" + f _ _ _ _ = return "/dev/null" result = checkWithIncludesAndSourcePath [("foo/lib", "echo $1")] f emptyCheckSpec { csScript = "#!/bin/bash\n# shellcheck source=lib\nsource kittens", csFilename = "dir/myscript", csCheckSourced = True } +prop_rcCanAllowExternalSources = result == [2086] + where + f "dir/myscript" (Just True) _ "mylib" = return "resolved/mylib" + f a b c d = error $ show ("Unexpected", a, b, c, d) + result = checkWithRcIncludesAndSourcePath "external-sources=true" [("resolved/mylib", "echo $1")] f emptyCheckSpec { + csScript = "#!/bin/bash\nsource mylib", + csFilename = "dir/myscript", + csCheckSourced = True + } + +prop_rcCanDenyExternalSources = result == [2086] + where + f "dir/myscript" (Just False) _ "mylib" = return "resolved/mylib" + f a b c d = error $ show ("Unexpected", a, b, c, d) + result = checkWithRcIncludesAndSourcePath "external-sources=false" [("resolved/mylib", "echo $1")] f emptyCheckSpec { + csScript = "#!/bin/bash\nsource mylib", + csFilename = "dir/myscript", + csCheckSourced = True + } + +prop_rcCanLeaveExternalSourcesUnspecified = result == [2086] + where + f "dir/myscript" Nothing _ "mylib" = return "resolved/mylib" + f a b c d = error $ show ("Unexpected", a, b, c, d) + result = checkWithRcIncludesAndSourcePath "" [("resolved/mylib", "echo $1")] f emptyCheckSpec { + csScript = "#!/bin/bash\nsource mylib", + csFilename = "dir/myscript", + csCheckSourced = True + } + +prop_fileCanDisableExternalSources = result == [2006, 2086] + where + f "dir/myscript" (Just True) _ "withExternal" = return "withExternal" + f "dir/myscript" (Just False) _ "withoutExternal" = return "withoutExternal" + f a b c d = error $ show ("Unexpected", a, b, c, d) + result = checkWithRcIncludesAndSourcePath "external-sources=true" [("withExternal", "echo $1"), ("withoutExternal", "_=`foo`")] f emptyCheckSpec { + csScript = "#!/bin/bash\ntrue\nsource withExternal\n# shellcheck external-sources=false\nsource withoutExternal", + csFilename = "dir/myscript", + csCheckSourced = True + } + +prop_fileCannotEnableExternalSources = result == [1144] + where + f "dir/myscript" Nothing _ "foo" = return "foo" + f a b c d = error $ show ("Unexpected", a, b, c, d) + result = checkWithRcIncludesAndSourcePath "" [("foo", "true")] f emptyCheckSpec { + csScript = "#!/bin/bash\n# shellcheck external-sources=true\nsource foo", + csFilename = "dir/myscript", + csCheckSourced = True + } + +prop_fileCannotEnableExternalSources2 = result == [1144] + where + f "dir/myscript" (Just False) _ "foo" = return "foo" + f a b c d = error $ show ("Unexpected", a, b, c, d) + result = checkWithRcIncludesAndSourcePath "external-sources=false" [("foo", "true")] f emptyCheckSpec { + csScript = "#!/bin/bash\n# shellcheck external-sources=true\nsource foo", + csFilename = "dir/myscript", + csCheckSourced = True + } + + return [] runTests = $quickCheckAll diff --git a/src/ShellCheck/Formatter/CheckStyle.hs b/src/ShellCheck/Formatter/CheckStyle.hs index f3fea88..c79ac21 100644 --- a/src/ShellCheck/Formatter/CheckStyle.hs +++ b/src/ShellCheck/Formatter/CheckStyle.hs @@ -48,7 +48,7 @@ outputResults cr sys = fileGroups = groupWith sourceFile comments outputGroup group = do let filename = sourceFile (head group) - result <- (siReadFile sys) filename + result <- siReadFile sys (Just True) filename let contents = either (const "") id result outputFile filename contents group diff --git a/src/ShellCheck/Formatter/Diff.hs b/src/ShellCheck/Formatter/Diff.hs index 83fb232..9e31780 100644 --- a/src/ShellCheck/Formatter/Diff.hs +++ b/src/ShellCheck/Formatter/Diff.hs @@ -90,7 +90,7 @@ reportResult foundIssues reportedIssues color result sys = do mapM_ output $ M.toList fixmap where output (name, fix) = do - file <- (siReadFile sys) name + file <- siReadFile sys (Just True) name case file of Right contents -> do putStrLn $ formatDoc color $ makeDiff name contents fix diff --git a/src/ShellCheck/Formatter/GCC.hs b/src/ShellCheck/Formatter/GCC.hs index 9c5fa5f..5106e4c 100644 --- a/src/ShellCheck/Formatter/GCC.hs +++ b/src/ShellCheck/Formatter/GCC.hs @@ -43,7 +43,7 @@ outputAll cr sys = mapM_ f groups f :: [PositionedComment] -> IO () f group = do let filename = sourceFile (head group) - result <- (siReadFile sys) filename + result <- siReadFile sys (Just True) filename let contents = either (const "") id result outputResult filename contents group diff --git a/src/ShellCheck/Formatter/JSON1.hs b/src/ShellCheck/Formatter/JSON1.hs index 7335d8c..54aad34 100644 --- a/src/ShellCheck/Formatter/JSON1.hs +++ b/src/ShellCheck/Formatter/JSON1.hs @@ -117,7 +117,7 @@ collectResult ref cr sys = mapM_ f groups f :: [PositionedComment] -> IO () f group = do let filename = sourceFile (head group) - result <- siReadFile sys filename + result <- siReadFile sys (Just True) filename let contents = either (const "") id result let comments' = makeNonVirtual comments contents modifyIORef ref (\x -> comments' ++ x) diff --git a/src/ShellCheck/Formatter/TTY.hs b/src/ShellCheck/Formatter/TTY.hs index 0d474d7..bb57894 100644 --- a/src/ShellCheck/Formatter/TTY.hs +++ b/src/ShellCheck/Formatter/TTY.hs @@ -121,7 +121,7 @@ outputResult options ref result sys = do outputForFile color sys comments = do let fileName = sourceFile (head comments) - result <- (siReadFile sys) fileName + result <- siReadFile sys (Just True) fileName let contents = either (const "") id result let fileLinesList = lines contents let lineCount = length fileLinesList diff --git a/src/ShellCheck/Interface.hs b/src/ShellCheck/Interface.hs index 87346a1..7528559 100644 --- a/src/ShellCheck/Interface.hs +++ b/src/ShellCheck/Interface.hs @@ -73,14 +73,18 @@ import qualified Data.Map as Map data SystemInterface m = SystemInterface { - -- | Read a file by filename, or return an error - siReadFile :: String -> m (Either ErrorMessage String), + -- | Given: + -- What annotations say about including external files (if anything) + -- A resolved filename from siFindSource + -- Read the file or return an error + siReadFile :: Maybe Bool -> String -> m (Either ErrorMessage String), -- | Given: -- the current script, + -- what annotations say about including external files (if anything) -- a list of source-path annotations in effect, -- and a sourced file, -- find the sourced file - siFindSource :: String -> [String] -> String -> m FilePath, + siFindSource :: String -> Maybe Bool -> [String] -> String -> m FilePath, -- | Get the configuration file (name, contents) for a filename siGetConfig :: String -> m (Maybe (FilePath, String)) } @@ -313,11 +317,11 @@ mockedSystemInterface files = SystemInterface { siGetConfig = const $ return Nothing } where - rf file = return $ + rf _ file = return $ case find ((== file) . fst) files of Nothing -> Left "File not included in mock." Just (_, contents) -> Right contents - fs _ _ file = return file + fs _ _ _ file = return file mockRcFile rcfile mock = mock { siGetConfig = const . return $ Just (".shellcheckrc", rcfile) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 45434a8..b59ebc2 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -987,9 +987,9 @@ prop_readAnnotation7 = isOk readAnnotation "# shellcheck disable=SC1000,SC2000-S readAnnotation = called "shellcheck directive" $ do try readAnnotationPrefix many1 linewhitespace - readAnnotationWithoutPrefix + readAnnotationWithoutPrefix True -readAnnotationWithoutPrefix = do +readAnnotationWithoutPrefix sandboxed = do values <- many1 readKey optional readAnyComment void linefeed <|> eof <|> do @@ -1035,6 +1035,21 @@ readAnnotationWithoutPrefix = do "This shell type is unknown. Use e.g. sh or bash." return [ShellOverride shell] + "external-sources" -> do + pos <- getPosition + value <- many1 letter + case value of + "true" -> + if sandboxed + then do + parseNoteAt pos ErrorC 1144 "external-sources can only be enabled in .shellcheckrc, not in individual files." + return [] + else return [ExternalSources True] + "false" -> return [ExternalSources False] + _ -> do + parseNoteAt pos ErrorC 1145 "Unknown external-sources value. Expected true/false." + return [] + _ -> do parseNoteAt keyPos WarningC 1107 "This directive is unknown. It will be ignored." anyChar `reluctantlyTill` whitespace @@ -2176,10 +2191,12 @@ readSource t@(T_Redirecting _ _ (T_SimpleCommand cmdId _ (cmd:file':rest'))) = d if filename == "/dev/null" -- always allow /dev/null then return (Right "", filename) else do + allAnnotations <- getCurrentAnnotations True currentScript <- Mr.asks currentFilename - paths <- mapMaybe getSourcePath <$> getCurrentAnnotations True - resolved <- system $ siFindSource sys currentScript paths filename - contents <- system $ siReadFile sys resolved + let paths = mapMaybe getSourcePath allAnnotations + let externalSources = listToMaybe $ mapMaybe getExternalSources allAnnotations + resolved <- system $ siFindSource sys currentScript externalSources paths filename + contents <- system $ siReadFile sys externalSources resolved return (contents, resolved) case input of Left err -> do @@ -2213,6 +2230,11 @@ readSource t@(T_Redirecting _ _ (T_SimpleCommand cmdId _ (cmd:file':rest'))) = d SourcePath x -> Just x _ -> Nothing + getExternalSources t = + case t of + ExternalSources b -> Just b + _ -> Nothing + -- If the word has a single expansion as the directory, try stripping it -- This affects `$foo/bar` but not `${foo}-dir/bar` or `/foo/$file` stripDynamicPrefix word = @@ -3202,7 +3224,7 @@ prop_readConfigKVs4 = isOk readConfigKVs "\n\n\n\n\t \n" prop_readConfigKVs5 = isOk readConfigKVs "# shellcheck accepts annotation-like comments in rc files\ndisable=1234" readConfigKVs = do anySpacingOrComment - annotations <- many (readAnnotationWithoutPrefix <* anySpacingOrComment) + annotations <- many (readAnnotationWithoutPrefix False <* anySpacingOrComment) eof return $ concat annotations anySpacingOrComment = From 09aa15c9b7bc33135178c7fb3338a870f7ce6a88 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 18 Sep 2021 12:50:01 -0700 Subject: [PATCH 491/763] Allow `disable=all` to disable all warnings (fixes #2323) --- CHANGELOG.md | 1 + shellcheck.1.md | 1 + src/ShellCheck/Checker.hs | 7 +++++++ src/ShellCheck/Parser.hs | 7 ++++++- 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4dc25f3..125d0c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ## Git (0.8.0) ### Added +- `disable=all` now conveniently disables all warnings - `external-sources=true` directive can be added to .shellcheckrc to make shellcheck behave as if `-x` was specified. - SC2286-SC2288: Warn when command name ends in a symbol like `/.)'"` diff --git a/shellcheck.1.md b/shellcheck.1.md index 070c3a4..1103acc 100644 --- a/shellcheck.1.md +++ b/shellcheck.1.md @@ -237,6 +237,7 @@ Valid keys are: The command can be a simple command like `echo foo`, or a compound command like a function definition, subshell block or loop. A range can be be specified with a dash, e.g. `disable=SC3000-SC4000` to exclude 3xxx. + All warnings can be disabled with `disable=all`. **enable** : Enable an optional check by name, as listed with **--list-optional**. diff --git a/src/ShellCheck/Checker.hs b/src/ShellCheck/Checker.hs index 514f97d..cce1063 100644 --- a/src/ShellCheck/Checker.hs +++ b/src/ShellCheck/Checker.hs @@ -306,6 +306,13 @@ prop_canDisableShebangWarning = null $ result csScript = "#shellcheck disable=SC2148\nfoo" } +prop_canDisableAllWarnings = result == [2086] + where + result = checkWithSpec [] emptyCheckSpec { + csFilename = "file.sh", + csScript = "#!/bin/sh\necho $1\n#shellcheck disable=all\necho `echo $1`" + } + prop_canDisableParseErrors = null $ result where result = checkWithSpec [] emptyCheckSpec { diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index b59ebc2..4f26a80 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -984,6 +984,7 @@ prop_readAnnotation4 = isWarning readAnnotation "# shellcheck cats=dogs disable= prop_readAnnotation5 = isOk readAnnotation "# shellcheck disable=SC2002 # All cats are precious\n" prop_readAnnotation6 = isOk readAnnotation "# shellcheck disable=SC1234 # shellcheck foo=bar\n" prop_readAnnotation7 = isOk readAnnotation "# shellcheck disable=SC1000,SC2000-SC3000,SC1001\n" +prop_readAnnotation8 = isOk readAnnotation "# shellcheck disable=all\n" readAnnotation = called "shellcheck directive" $ do try readAnnotationPrefix many1 linewhitespace @@ -1004,8 +1005,12 @@ readAnnotationWithoutPrefix sandboxed = do key <- many1 (letter <|> char '-') char '=' <|> fail "Expected '=' after directive key" annotations <- case key of - "disable" -> readRange `sepBy` char ',' + "disable" -> readElement `sepBy` char ',' where + readElement = readRange <|> readAll + readAll = do + string "all" + return $ DisableComment 0 1000000 readRange = do from <- readCode to <- choice [ char '-' *> readCode, return $ from+1 ] From 253650706086662c8d655e80b2e3d0e50a3df837 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 18 Sep 2021 17:43:55 -0700 Subject: [PATCH 492/763] Remove SC1004 (fixes #2326) --- CHANGELOG.md | 3 +++ src/ShellCheck/Parser.hs | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 125d0c5..fef7680 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,9 @@ - Quote warnings are now emitted for declaration utilities in sh - Leading `_` can now be used to suppress warnings about unused variables +### Removed +- SC1003: Literal backslash+linefeed in '' was found to be usually correct + ## v0.7.2 - 2021-04-19 ### Added diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 4f26a80..30bbb69 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -1511,7 +1511,6 @@ readSingleEscaped = do case x of '\'' -> parseProblemAt pos InfoC 1003 "Want to escape a single quote? echo 'This is how it'\\''s done'."; - '\n' -> parseProblemAt pos InfoC 1004 "This backslash+linefeed is literal. Break outside single quotes if you just want to break the line." _ -> return () return [s] From 8012f6761d1d32b5a6d26f86b50ae639457f1b3d Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 18 Sep 2021 17:59:30 -0700 Subject: [PATCH 493/763] Suppress SC2094 when both are input redirections (fixes #2325) --- src/ShellCheck/Analytics.hs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 34bca94..b21a821 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -791,6 +791,7 @@ prop_checkRedirectToSame5 = verifyNot checkRedirectToSame "foo > bar 2> bar" prop_checkRedirectToSame6 = verifyNot checkRedirectToSame "echo foo > foo" prop_checkRedirectToSame7 = verifyNot checkRedirectToSame "sed 's/foo/bar/g' file | sponge file" prop_checkRedirectToSame8 = verifyNot checkRedirectToSame "while read -r line; do _=\"$fname\"; done <\"$fname\"" +prop_checkRedirectToSame9 = verifyNot checkRedirectToSame "while read -r line; do cat < \"$fname\"; done <\"$fname\"" checkRedirectToSame params s@(T_Pipeline _ _ list) = mapM_ (\l -> (mapM_ (\x -> doAnalysis (checkOccurrences x) l) (getAllRedirs list))) list where @@ -799,6 +800,7 @@ checkRedirectToSame params s@(T_Pipeline _ _ list) = checkOccurrences t@(T_NormalWord exceptId x) u@(T_NormalWord newId y) | exceptId /= newId && x == y + && not (isInput t && isInput u) && not (isOutput t && isOutput u) && not (special t) && not (any isHarmlessCommand [t,u]) @@ -817,6 +819,13 @@ checkRedirectToSame params s@(T_Pipeline _ _ list) = _ -> [] getRedirs _ = [] special x = "/dev/" `isPrefixOf` concat (oversimplify x) + isInput t = + case drop 1 $ getPath (parentMap params) t of + T_IoFile _ op _:_ -> + case op of + T_Less _ -> True + _ -> False + _ -> False isOutput t = case drop 1 $ getPath (parentMap params) t of T_IoFile _ op _:_ -> From b044f5b23ab79b40f6885f28eafd82666b9ee7d4 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 18 Sep 2021 18:49:58 -0700 Subject: [PATCH 494/763] Don't trigger SC2140 on ${x+"a" "b"} (fixes #2265) --- src/ShellCheck/Analytics.hs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index b21a821..49a6557 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1763,6 +1763,7 @@ prop_checkInexplicablyUnquoted6 = verifyNot checkInexplicablyUnquoted "\"$dir\"s prop_checkInexplicablyUnquoted7 = verifyNot checkInexplicablyUnquoted "${dir/\"foo\"/\"bar\"}" prop_checkInexplicablyUnquoted8 = verifyNot checkInexplicablyUnquoted " 'foo'\\\n 'bar'" prop_checkInexplicablyUnquoted9 = verifyNot checkInexplicablyUnquoted "[[ $x =~ \"foo\"(\"bar\"|\"baz\") ]]" +prop_checkInexplicablyUnquoted10 = verifyNot checkInexplicablyUnquoted "cmd ${x+--name=\"$x\" --output=\"$x.out\"}" checkInexplicablyUnquoted params (T_NormalWord id tokens) = mapM_ check (tails tokens) where check (T_SingleQuoted _ _:T_Literal id str:_) @@ -1774,19 +1775,20 @@ checkInexplicablyUnquoted params (T_NormalWord id tokens) = mapM_ check (tails t T_DollarExpansion id _ -> warnAboutExpansion id T_DollarBraced id _ _ -> warnAboutExpansion id T_Literal id s - | not (quotesSingleThing a && quotesSingleThing b || isRegex (getPath (parentMap params) trapped)) -> + | not (quotesSingleThing a && quotesSingleThing b || isSpecial (getPath (parentMap params) trapped)) -> warnAboutLiteral id _ -> return () check _ = return () -- Regexes for [[ .. =~ re ]] are parsed with metacharacters like ()| as unquoted - -- literals, so avoid overtriggering on these. - isRegex t = + -- literals. The same is true for ${x+"foo" "bar"}. Avoid overtriggering on these. + isSpecial t = case t of (T_Redirecting {} : _) -> False + T_DollarBraced {} : _ -> True (a:(TC_Binary _ _ "=~" lhs rhs):rest) -> getId a == getId rhs - _:rest -> isRegex rest + _:rest -> isSpecial rest _ -> False -- If the surrounding quotes quote single things, like "$foo"_and_then_some_"$stuff", From e7df718724341e1ff9ea010ab867afc0376f67b0 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 18 Sep 2021 19:22:46 -0700 Subject: [PATCH 495/763] Strip lines containing "STRIP" from ./striptests --- src/ShellCheck/Analytics.hs | 6 +++--- src/ShellCheck/Formatter/Diff.hs | 3 --- src/ShellCheck/Parser.hs | 10 +++++----- striptests | 1 + 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 49a6557..de59be1 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -42,7 +42,7 @@ import Data.List import Data.Maybe import Data.Ord import Data.Semigroup -import Debug.Trace +import Debug.Trace -- STRIP import qualified Data.Map.Strict as Map import Test.QuickCheck.All (forAllProperties) import Test.QuickCheck.Test (quickCheckWithResult, stdArgs, maxSuccess) @@ -1011,8 +1011,8 @@ checkStderrRedirect params redir@(T_Redirecting _ [ checkStderrRedirect _ _ = return () -lt x = trace ("Tracing " ++ show x) x -ltt t = trace ("Tracing " ++ show t) +lt x = trace ("Tracing " ++ show x) x -- STRIP +ltt t = trace ("Tracing " ++ show t) -- STRIP prop_checkSingleQuotedVariables = verify checkSingleQuotedVariables "echo '$foo'" diff --git a/src/ShellCheck/Formatter/Diff.hs b/src/ShellCheck/Formatter/Diff.hs index 9e31780..197b3af 100644 --- a/src/ShellCheck/Formatter/Diff.hs +++ b/src/ShellCheck/Formatter/Diff.hs @@ -38,9 +38,6 @@ import System.FilePath import Test.QuickCheck -import Debug.Trace -ltt x = trace (show x) x - format :: FormatterOptions -> IO Formatter format options = do foundIssues <- newIORef False diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 30bbb69..e269b4d 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -37,7 +37,7 @@ import Data.Functor import Data.List (isPrefixOf, isInfixOf, isSuffixOf, partition, sortBy, intercalate, nub, find) import Data.Maybe import Data.Monoid -import Debug.Trace +import Debug.Trace -- STRIP import GHC.Exts (sortWith) import Prelude hiding (readList) import System.IO @@ -3372,13 +3372,13 @@ parsesCleanly parser string = runIdentity $ do -- For printf debugging: print the value of an expression -- Example: return $ dump $ T_Literal id [c] -dump :: Show a => a -> a -dump x = trace (show x) x +dump :: Show a => a -> a -- STRIP +dump x = trace (show x) x -- STRIP -- Like above, but print a specific expression: -- Example: return $ dumps ("Returning: " ++ [c]) $ T_Literal id [c] -dumps :: Show x => x -> a -> a -dumps t = trace (show t) +dumps :: Show x => x -> a -> a -- STRIP +dumps t = trace (show t) -- STRIP parseWithNotes parser = do item <- parser diff --git a/striptests b/striptests index c68c16a..6e64607 100755 --- a/striptests +++ b/striptests @@ -29,6 +29,7 @@ detestify() { state = 0; } + /STRIP/ { next; } /LANGUAGE TemplateHaskell/ { next; } /^import.*Test\./ { next; } From db4701d8b54fa1bb980bd24cfe2d1fe446278c6b Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 18 Sep 2021 19:32:12 -0700 Subject: [PATCH 496/763] Add a `setgitversion` script to update the version string with git --- .github/workflows/build.yml | 3 +++ setgitversion | 11 +++++++++++ src/ShellCheck/Data.hs | 2 +- 3 files changed, 15 insertions(+), 1 deletion(-) create mode 100755 setgitversion diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0acece2..e28a43f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,9 +16,12 @@ jobs: - name: Checkout repository uses: actions/checkout@v2 + with: + fetch-depth: 0 - name: Package Source run: | + ./setgitversion mkdir source cabal sdist mv dist-newstyle/sdist/*.tar.gz source/source.tar.gz diff --git a/setgitversion b/setgitversion new file mode 100755 index 0000000..3afad61 --- /dev/null +++ b/setgitversion @@ -0,0 +1,11 @@ +#!/bin/sh -xe +# This script hardcodes the `git describe` version as ShellCheck's version number. +# This is done to allow shellcheck --version to differ from the cabal version when +# building git snapshots. + +file="src/ShellCheck/Data.hs" +test -e "$file" +tmp=$(mktemp) +version=$(git describe) +sed -e "s/=.*VERSIONSTRING.*/= \"$version\" -- VERSIONSTRING, DO NOT SUBMIT/" "$file" > "$tmp" +mv "$tmp" "$file" diff --git a/src/ShellCheck/Data.hs b/src/ShellCheck/Data.hs index fb4a1e4..793a4de 100644 --- a/src/ShellCheck/Data.hs +++ b/src/ShellCheck/Data.hs @@ -4,7 +4,7 @@ import ShellCheck.Interface import Data.Version (showVersion) import Paths_ShellCheck (version) -shellcheckVersion = showVersion version +shellcheckVersion = showVersion version -- VERSIONSTRING internalVariables = [ -- Generic From 3a296cd788cdb9313ed2fc2aa5e86a876269d407 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 19 Sep 2021 12:27:16 -0700 Subject: [PATCH 497/763] The removed check was SC1004, not SC1003 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fef7680..48fbfeb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,7 +29,7 @@ - Leading `_` can now be used to suppress warnings about unused variables ### Removed -- SC1003: Literal backslash+linefeed in '' was found to be usually correct +- SC1004: Literal backslash+linefeed in '' was found to be usually correct ## v0.7.2 - 2021-04-19 From ad92cb411291631b5f1d06df083b92f02128b33d Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 25 Sep 2021 19:46:27 -0700 Subject: [PATCH 498/763] Disable UUOC for cat with unquoted variable (fixes #2333) --- src/ShellCheck/ASTLib.hs | 13 ++++++------- src/ShellCheck/Analytics.hs | 6 ++++-- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 20e5be4..5c13697 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -287,14 +287,14 @@ isArrayExpansion (T_DollarBraced _ _ l) = isArrayExpansion _ = False -- Is it possible that this arg becomes multiple args? -mayBecomeMultipleArgs t = willBecomeMultipleArgs t || f t +mayBecomeMultipleArgs t = willBecomeMultipleArgs t || f False t where - f (T_DollarBraced _ _ l) = + f quoted (T_DollarBraced _ _ l) = let string = concat $ oversimplify l in - "!" `isPrefixOf` string - f (T_DoubleQuoted _ parts) = any f parts - f (T_NormalWord _ parts) = any f parts - f _ = False + not quoted || "!" `isPrefixOf` string + f quoted (T_DoubleQuoted _ parts) = any (f True) parts + f quoted (T_NormalWord _ parts) = any (f quoted) parts + f _ _ = False -- Is it certain that this word will becomes multiple words? willBecomeMultipleArgs t = willConcatInAssignment t || f t @@ -302,7 +302,6 @@ willBecomeMultipleArgs t = willConcatInAssignment t || f t f T_Extglob {} = True f T_Glob {} = True f T_BraceExpansion {} = True - f (T_DoubleQuoted _ parts) = any f parts f (T_NormalWord _ parts) = any f parts f _ = False diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index de59be1..749a93d 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -506,7 +506,9 @@ checkWrongArithmeticAssignment _ _ = return () prop_checkUuoc1 = verify checkUuoc "cat foo | grep bar" prop_checkUuoc2 = verifyNot checkUuoc "cat * | grep bar" -prop_checkUuoc3 = verify checkUuoc "cat $var | grep bar" +prop_checkUuoc3 = verify checkUuoc "cat \"$var\" | grep bar" +prop_checkUuoc3b = verifyNot checkUuoc "cat $var | grep bar" +prop_checkUuoc3c = verifyNot checkUuoc "cat \"${!var}\" | grep bar" prop_checkUuoc4 = verifyNot checkUuoc "cat $var" prop_checkUuoc5 = verifyNot checkUuoc "cat \"$@\"" prop_checkUuoc6 = verifyNot checkUuoc "cat -n | grep bar" @@ -659,7 +661,7 @@ prop_checkForInQuoted7 = verify checkForInQuoted "for f in ls, grep, mv; do true prop_checkForInQuoted8 = verify checkForInQuoted "for f in 'ls', 'grep', 'mv'; do true; done" prop_checkForInQuoted9 = verifyNot checkForInQuoted "for f in 'ls,' 'grep,' 'mv'; do true; done" checkForInQuoted _ (T_ForIn _ f [T_NormalWord _ [word@(T_DoubleQuoted id list)]] _) - | any (\x -> willSplit x && not (mayBecomeMultipleArgs x)) list + | any willSplit list && not (mayBecomeMultipleArgs word) || maybe False wouldHaveBeenGlob (getLiteralString word) = err id 2066 "Since you double quoted this, it will not word split, and the loop will only run once." checkForInQuoted _ (T_ForIn _ f [T_NormalWord _ [T_SingleQuoted id _]] _) = From fac97a5301b39306e6f26cb6d4fadbbaa4e6f8b0 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 25 Sep 2021 20:23:58 -0700 Subject: [PATCH 499/763] Don't emit SC2140 when trapped string is /, = or : (fixes #2334) --- src/ShellCheck/Analytics.hs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 749a93d..bb95cf4 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1766,6 +1766,8 @@ prop_checkInexplicablyUnquoted7 = verifyNot checkInexplicablyUnquoted "${dir/\"f prop_checkInexplicablyUnquoted8 = verifyNot checkInexplicablyUnquoted " 'foo'\\\n 'bar'" prop_checkInexplicablyUnquoted9 = verifyNot checkInexplicablyUnquoted "[[ $x =~ \"foo\"(\"bar\"|\"baz\") ]]" prop_checkInexplicablyUnquoted10 = verifyNot checkInexplicablyUnquoted "cmd ${x+--name=\"$x\" --output=\"$x.out\"}" +prop_checkInexplicablyUnquoted11 = verifyNot checkInexplicablyUnquoted "echo \"foo\"/\"bar\"" +prop_checkInexplicablyUnquoted12 = verifyNot checkInexplicablyUnquoted "declare \"foo\"=\"bar\"" checkInexplicablyUnquoted params (T_NormalWord id tokens) = mapM_ check (tails tokens) where check (T_SingleQuoted _ _:T_Literal id str:_) @@ -1777,7 +1779,10 @@ checkInexplicablyUnquoted params (T_NormalWord id tokens) = mapM_ check (tails t T_DollarExpansion id _ -> warnAboutExpansion id T_DollarBraced id _ _ -> warnAboutExpansion id T_Literal id s - | not (quotesSingleThing a && quotesSingleThing b || isSpecial (getPath (parentMap params) trapped)) -> + | not (quotesSingleThing a && quotesSingleThing b + || s `elem` ["=", ":", "/"] + || isSpecial (getPath (parentMap params) trapped) + ) -> warnAboutLiteral id _ -> return () From 093df8cb2448a3b2f352b4e3105d2af45ccf3206 Mon Sep 17 00:00:00 2001 From: Christian Nassif-Haynes Date: Mon, 6 Sep 2021 05:52:34 +1000 Subject: [PATCH 500/763] Add extra checks for masked return codes --- src/ShellCheck/Analytics.hs | 109 ++++++++++++++++++++++++++++++ src/ShellCheck/AnalyzerLib.hs | 18 +++++ src/ShellCheck/Checks/Commands.hs | 2 - src/ShellCheck/Data.hs | 2 + 4 files changed, 129 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index de59be1..a09e804 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -260,6 +260,13 @@ optionalTreeChecks = [ cdPositive = "set -e; func() { cp *.txt ~/backup; rm *.txt; }; func && echo ok", cdNegative = "set -e; func() { cp *.txt ~/backup; rm *.txt; }; func; echo ok" }, checkSetESuppressed) + + ,(newCheckDescription { + cdName = "check-extra-masked-returns", + cdDescription = "Check for additional cases where exit codes are masked", + cdPositive = "rm -r \"$(get_chroot_dir)/home\"", + cdNegative = "set -e; dir=\"$(get_chroot_dir)\"; rm -r \"$dir/home\"" + }, checkExtraMaskedReturns) ] optionalCheckMap :: Map.Map String (Parameters -> Token -> [TokenComment]) @@ -4734,5 +4741,107 @@ checkSetESuppressed params t = isIn t cmds = getId t `elem` map getId cmds +prop_checkExtraMaskedReturns1 = verifyTree checkExtraMaskedReturns "cat < <(ls)" +prop_checkExtraMaskedReturns2 = verifyTree checkExtraMaskedReturns "read -ra arr <(ls)" +prop_checkExtraMaskedReturns3 = verifyTree checkExtraMaskedReturns "ls >(cat)" +prop_checkExtraMaskedReturns4 = verifyTree checkExtraMaskedReturns "false | true" +prop_checkExtraMaskedReturns5 = verifyNotTree checkExtraMaskedReturns "set -o pipefail; false | true" +prop_checkExtraMaskedReturns6 = verifyNotTree checkExtraMaskedReturns "false | true || true" +prop_checkExtraMaskedReturns7 = verifyTree checkExtraMaskedReturns "true $(false)" +prop_checkExtraMaskedReturns8 = verifyTree checkExtraMaskedReturns "x=$(false)$(true)" +prop_checkExtraMaskedReturns9 = verifyNotTree checkExtraMaskedReturns "x=$(false)true" +prop_checkExtraMaskedReturns10 = verifyTree checkExtraMaskedReturns "x=`false``false`" +prop_checkExtraMaskedReturns11 = verifyTree checkExtraMaskedReturns "x=\"$(false)$(true)\"" +prop_checkExtraMaskedReturns12 = verifyTree checkExtraMaskedReturns "x=\"$(false)\"\"$(true)\"" +prop_checkExtraMaskedReturns13 = verifyTree checkExtraMaskedReturns "true <<<$(false)" +prop_checkExtraMaskedReturns14 = verifyNotTree checkExtraMaskedReturns "echo asdf | false" +prop_checkExtraMaskedReturns15 = verifyNotTree checkExtraMaskedReturns "readonly x=$(false)" +prop_checkExtraMaskedReturns16 = verifyTree checkExtraMaskedReturns "readarray -t files < <(ls)" +prop_checkExtraMaskedReturns17 = verifyNotTree checkExtraMaskedReturns "x=( $(false) false )" +prop_checkExtraMaskedReturns18 = verifyTree checkExtraMaskedReturns "x=( $(false) $(false) )" +prop_checkExtraMaskedReturns19 = verifyNotTree checkExtraMaskedReturns "x=( $(false) [4]=false )" +prop_checkExtraMaskedReturns20 = verifyTree checkExtraMaskedReturns "x=( $(false) [4]=$(false) )" +prop_checkExtraMaskedReturns21 = verifyTree checkExtraMaskedReturns "cat << foo\n $(false)\nfoo" +prop_checkExtraMaskedReturns22 = verifyTree checkExtraMaskedReturns "[[ $(false) ]]" +prop_checkExtraMaskedReturns23 = verifyNotTree checkExtraMaskedReturns "x=$(false) y=z" +prop_checkExtraMaskedReturns24 = verifyNotTree checkExtraMaskedReturns "x=$(( $(date +%s) ))" +prop_checkExtraMaskedReturns25 = verifyTree checkExtraMaskedReturns "echo $(( $(date +%s) ))" +prop_checkExtraMaskedReturns26 = verifyNotTree checkExtraMaskedReturns "x=( $(false) )" +prop_checkExtraMaskedReturns27 = verifyTree checkExtraMaskedReturns "x=$(false) false" +prop_checkExtraMaskedReturns28 = verifyTree checkExtraMaskedReturns "x=$(false) y=$(false)" +prop_checkExtraMaskedReturns29 = verifyNotTree checkExtraMaskedReturns "false < <(set -e)" +prop_checkExtraMaskedReturns30 = verifyNotTree checkExtraMaskedReturns "false < <(shopt -s cdspell)" +prop_checkExtraMaskedReturns31 = verifyNotTree checkExtraMaskedReturns "false < <(dirname \"${BASH_SOURCE[0]}\")" +prop_checkExtraMaskedReturns32 = verifyNotTree checkExtraMaskedReturns "false < <(basename \"${BASH_SOURCE[0]}\")" +prop_checkExtraMaskedReturns33 = verifyNotTree checkExtraMaskedReturns "{ false || true; } | true" +prop_checkExtraMaskedReturns34 = verifyNotTree checkExtraMaskedReturns "{ false || :; } | true" +checkExtraMaskedReturns params t = runNodeAnalysis findMaskingNodes params t + where + findMaskingNodes _ (T_Arithmetic _ list) = findMaskedNodesInList [list] + findMaskingNodes _ (T_Array _ list) = findMaskedNodesInList $ allButLastSimpleCommands list + findMaskingNodes _ (T_Condition _ _ condition) = findMaskedNodesInList [condition] + findMaskingNodes _ (T_DoubleQuoted _ list) = findMaskedNodesInList $ allButLastSimpleCommands list + findMaskingNodes _ (T_HereDoc _ _ _ _ list) = findMaskedNodesInList list + findMaskingNodes _ (T_HereString _ word) = findMaskedNodesInList [word] + findMaskingNodes _ (T_NormalWord _ parts) = findMaskedNodesInList $ allButLastSimpleCommands parts + findMaskingNodes _ (T_Pipeline _ _ cmds) | not (hasPipefail params) = findMaskedNodesInList $ allButLastSimpleCommands cmds + findMaskingNodes _ (T_ProcSub _ _ list) = findMaskedNodesInList list + findMaskingNodes _ (T_SimpleCommand _ assigns (_:args)) = findMaskedNodesInList $ assigns ++ args + findMaskingNodes _ (T_SimpleCommand _ assigns []) = findMaskedNodesInList $ allButLastSimpleCommands assigns + findMaskingNodes _ _ = return () + + findMaskedNodesInList = mapM_ (doAnalysis findMaskedNodes) + + isMaskedNode t = not (isHarmlessCommand t || isCheckedElsewhere t || isMaskDeliberate t) + findMaskedNodes t@(T_SimpleCommand _ _ (_:_)) = when (isMaskedNode t) $ inform t + findMaskedNodes t@T_Condition {} = when (isMaskedNode t) $ inform t + findMaskedNodes _ = return () + + containsSimpleCommand t = isNothing $ doAnalysis go t + where + go t = case t of + T_SimpleCommand {} -> fail "" + _ -> return () + + allButLastSimpleCommands cmds = + if null simpleCommands then [] else init simpleCommands + where + simpleCommands = filter containsSimpleCommand cmds + + inform t = info (getId t) 2312 ("Consider invoking this command " + ++ "separately to avoid masking its return value (or use '|| true' " + ++ "to ignore).") + + isMaskDeliberate t = hasParent isOrIf t + where + isOrIf _ (T_OrIf _ _ (T_Pipeline _ _ [T_Redirecting _ _ cmd])) + = getCommandBasename cmd `elem` [Just "true", Just ":"] + isOrIf _ _ = False + + isCheckedElsewhere t = hasParent isDeclaringCommand t + where + isDeclaringCommand t _ = fromMaybe False $ do + basename <- getCommandBasename t + return $ basename `elem` declaringCommands + + isHarmlessCommand t = fromMaybe False $ do + basename <- getCommandBasename t + return $ basename `elem` [ + "echo" + ,"basename" + ,"dirname" + ,"printf" + ,"set" + ,"shopt" + ] + + parentChildPairs t = go $ parents params t + where + go (child:parent:rest) = (parent, child):go (parent:rest) + go _ = [] + + hasParent pred t = any (uncurry pred) (parentChildPairs t) + + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 9cb6cd2..be54fbb 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -83,6 +83,8 @@ data Parameters = Parameters { hasInheritErrexit :: Bool, -- Whether this script has 'set -e' anywhere. hasSetE :: Bool, + -- Whether this script has 'set -o pipefail' anywhere. + hasPipefail :: Bool, -- A linear (bad) analysis of data flow variableFlow :: [StackData], -- A map from Id to parent Token @@ -204,6 +206,12 @@ makeParameters spec = Dash -> True Sh -> True Ksh -> False, + hasPipefail = + case shellType params of + Bash -> containsPipefail root + Dash -> True + Sh -> True + Ksh -> containsPipefail root, shellTypeSpecified = isJust (asShellType spec) || isJust (asFallbackShell spec), parentMap = getParentTree root, variableFlow = getVariableFlow params root, @@ -226,6 +234,16 @@ containsSetE root = isNothing $ doAnalysis (guard . not . isSetE) root _ -> False re = mkRegex "[[:space:]]-[^-]*e" +containsPipefail root = isNothing $ doAnalysis (guard . not . isPipefail) root + where + isPipefail t = + case t of + T_SimpleCommand {} -> + t `isUnqualifiedCommand` "set" && + ("pipefail" `elem` oversimplify t || + "o" `elem` map snd (getAllFlags t)) + _ -> False + containsShopt shopt root = isNothing $ doAnalysis (guard . not . isShoptLastPipe) root where diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index d0ada73..77bdbf6 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -101,8 +101,6 @@ commandChecks = [ ++ map checkArgComparison declaringCommands ++ map checkMaskedReturns declaringCommands -declaringCommands = ["local", "declare", "export", "readonly", "typeset", "let"] - optionalChecks = map fst optionalCommandChecks optionalCommandChecks :: [(CheckDescription, CommandCheck)] diff --git a/src/ShellCheck/Data.hs b/src/ShellCheck/Data.hs index 793a4de..e22b424 100644 --- a/src/ShellCheck/Data.hs +++ b/src/ShellCheck/Data.hs @@ -138,3 +138,5 @@ shellForExecutable name = _ -> Nothing flagsForRead = "sreu:n:N:i:p:a:t:" + +declaringCommands = ["local", "declare", "export", "readonly", "typeset", "let"] From 6f7eee4a2779ced87334c4962b64241f83fdf7e0 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 2 Oct 2021 12:58:28 -0700 Subject: [PATCH 501/763] Mention check-extra-masked-returns in changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 48fbfeb..f7be144 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,8 @@ - `disable=all` now conveniently disables all warnings - `external-sources=true` directive can be added to .shellcheckrc to make shellcheck behave as if `-x` was specified. +- Optional `check-extra-masked-returns` for pointing out commands with + suppressed exit codes (SC2312). - SC2286-SC2288: Warn when command name ends in a symbol like `/.)'"` - SC2289: Warn when command name contains tabs or linefeeds - SC2291: Warn about repeated unquoted spaces between words in echo From 38251abe26d952e4602d819c74fcdba46af4b066 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 7 Oct 2021 17:14:41 -0700 Subject: [PATCH 502/763] Add suggestion level in text for TTY output (fixes #2339) --- CHANGELOG.md | 1 + src/ShellCheck/Formatter/TTY.hs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f7be144..1741552 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ - SC2181 now only triggers on single condition tests like `[ $? = 0 ]`. - Quote warnings are now emitted for declaration utilities in sh - Leading `_` can now be used to suppress warnings about unused variables +- TTY output now includes warning level in text as well as color ### Removed - SC1004: Literal backslash+linefeed in '' was found to be usually correct diff --git a/src/ShellCheck/Formatter/TTY.hs b/src/ShellCheck/Formatter/TTY.hs index bb57894..8dd90d4 100644 --- a/src/ShellCheck/Formatter/TTY.hs +++ b/src/ShellCheck/Formatter/TTY.hs @@ -174,7 +174,7 @@ showFixedString color comments lineNum fileLines = cuteIndent :: PositionedComment -> String cuteIndent comment = replicate (fromIntegral $ colNo comment - 1) ' ' ++ - makeArrow ++ " " ++ code (codeNo comment) ++ ": " ++ messageText comment + makeArrow ++ " " ++ code (codeNo comment) ++ " (" ++ severityText comment ++ "): " ++ messageText comment where arrow n = '^' : replicate (fromIntegral $ n-2) '-' ++ "^" makeArrow = From 05bdeae3abc539e27246f761f8184f1df7ee9878 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 7 Oct 2021 17:26:08 -0700 Subject: [PATCH 503/763] Mention require-double-brackets in CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1741552..c98254f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ shellcheck behave as if `-x` was specified. - Optional `check-extra-masked-returns` for pointing out commands with suppressed exit codes (SC2312). +- Optional `require-double-brackets` for recommending \[\[ ]] (SC2292). - SC2286-SC2288: Warn when command name ends in a symbol like `/.)'"` - SC2289: Warn when command name contains tabs or linefeeds - SC2291: Warn about repeated unquoted spaces between words in echo From 205ba429b3d8afee08e6e9f5ff504fc8aa62a9c8 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 7 Oct 2021 18:50:44 -0700 Subject: [PATCH 504/763] Warn about `read foo[i]` expanding as glob (fixes #2345) --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 9 +++++++-- src/ShellCheck/Checks/Commands.hs | 18 ++++++++++++++++-- 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c98254f..a1456a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ - SC2304-SC2306: Warn about unquoted globs in expr arguments - SC2307: Warn about insufficient number of arguments to expr - SC2308: Suggest other approaches for non-standard expr extensions +- SC2313: Warn about `read` with unquoted, array indexed variable ### Fixed - SC2102 about repetitions in ranges no longer triggers on [[ -v arr[xx] ]] diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 3c99cea..787a501 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2689,12 +2689,13 @@ prop_checkCharRangeGlob1 = verify checkCharRangeGlob "ls *[:digit:].jpg" prop_checkCharRangeGlob2 = verifyNot checkCharRangeGlob "ls *[[:digit:]].jpg" prop_checkCharRangeGlob3 = verify checkCharRangeGlob "ls [10-15]" prop_checkCharRangeGlob4 = verifyNot checkCharRangeGlob "ls [a-zA-Z]" -prop_checkCharRangeGlob5 = verifyNot checkCharRangeGlob "tr -d [a-zA-Z]" -- tr has 2060 +prop_checkCharRangeGlob5 = verifyNot checkCharRangeGlob "tr -d [aa]" -- tr has 2060 prop_checkCharRangeGlob6 = verifyNot checkCharRangeGlob "[[ $x == [!!]* ]]" prop_checkCharRangeGlob7 = verifyNot checkCharRangeGlob "[[ -v arr[keykey] ]]" prop_checkCharRangeGlob8 = verifyNot checkCharRangeGlob "[[ arr[keykey] -gt 1 ]]" +prop_checkCharRangeGlob9 = verifyNot checkCharRangeGlob "read arr[keykey]" -- tr has 2313 checkCharRangeGlob p t@(T_Glob id str) | - isCharClass str && not (isParamTo (parentMap p) "tr" t) && not (isDereferenced t) = + isCharClass str && not isIgnoredCommand && not (isDereferenced t) = if ":" `isPrefixOf` contents && ":" `isSuffixOf` contents && contents /= ":" @@ -2712,6 +2713,10 @@ checkCharRangeGlob p t@(T_Glob id str) | '^':rest -> rest x -> x + isIgnoredCommand = fromMaybe False $ do + cmd <- getClosestCommand (parentMap p) t + return $ isCommandMatch cmd (`elem` ["tr", "read"]) + -- Check if this is a dereferencing context like [[ -v array[operandhere] ]] isDereferenced = fromMaybe False . msum . map isDereferencingOp . getPath (parentMap p) isDereferencingOp t = diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 77bdbf6..519ba4f 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -790,6 +790,7 @@ prop_checkReadExpansions5 = verify checkReadExpansions "read \"$var\"" prop_checkReadExpansions6 = verify checkReadExpansions "read -a $var" prop_checkReadExpansions7 = verifyNot checkReadExpansions "read $1" prop_checkReadExpansions8 = verifyNot checkReadExpansions "read ${var?}" +prop_checkReadExpansions9 = verify checkReadExpansions "read arr[val]" checkReadExpansions = CommandCheck (Exactly "read") check where options = getGnuOpts flagsForRead @@ -797,13 +798,26 @@ checkReadExpansions = CommandCheck (Exactly "read") check opts <- options $ arguments cmd return [y | (x,(_, y)) <- opts, null x || x == "a"] - check cmd = mapM_ warning $ getVars cmd - warning t = sequence_ $ do + check cmd = do + mapM_ dollarWarning $ getVars cmd + mapM_ arrayWarning $ arguments cmd + + dollarWarning t = sequence_ $ do name <- getSingleUnmodifiedBracedString t guard $ isVariableName name -- e.g. not $1 return . warn (getId t) 2229 $ "This does not read '" ++ name ++ "'. Remove $/${} for that, or use ${var?} to quiet." + arrayWarning word = + when (any isUnquotedBracket $ getWordParts word) $ + warn (getId word) 2313 $ + "Quote array indices to avoid them expanding as globs." + + isUnquotedBracket t = + case t of + T_Glob _ ('[':_) -> True + _ -> False + -- Return the single variable expansion that makes up this word, if any. -- e.g. $foo -> $foo, "$foo"'' -> $foo , "hello $name" -> Nothing getSingleUnmodifiedBracedString :: Token -> Maybe String From 3aedda766de75b21b6b6eff58da3881c41e4fa16 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 9 Oct 2021 11:40:52 -0700 Subject: [PATCH 505/763] For `while getopts; do case ..` checks, make sure variable matches --- src/ShellCheck/Checks/Commands.hs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 519ba4f..edc41ed 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -959,15 +959,24 @@ prop_checkWhileGetoptsCase2 = verify checkWhileGetoptsCase "while getopts 'a:' x prop_checkWhileGetoptsCase3 = verifyNot checkWhileGetoptsCase "while getopts 'a:b' x; do case $x in a) foo;; b) bar;; *) :;esac; done" prop_checkWhileGetoptsCase4 = verifyNot checkWhileGetoptsCase "while getopts 'a:123' x; do case $x in a) foo;; [0-9]) bar;; esac; done" prop_checkWhileGetoptsCase5 = verifyNot checkWhileGetoptsCase "while getopts 'a:' x; do case $x in a) foo;; \\?) bar;; *) baz;; esac; done" +prop_checkWhileGetoptsCase6 = verifyNot checkWhileGetoptsCase "while getopts 'a:b' x; do case $y in a) foo;; esac; done" +prop_checkWhileGetoptsCase7 = verifyNot checkWhileGetoptsCase "while getopts 'a:b' x; do case x$x in xa) foo;; xb) foo;; esac; done" checkWhileGetoptsCase = CommandCheck (Exactly "getopts") f where f :: Token -> Analysis - f t@(T_SimpleCommand _ _ (cmd:arg1:_)) = do + f t@(T_SimpleCommand _ _ (cmd:arg1:name:_)) = do path <- getPathM t sequence_ $ do options <- getLiteralString arg1 + getoptsVar <- getLiteralString name (T_WhileExpression _ _ body) <- findFirst whileLoop path - caseCmd <- mapMaybe findCase body !!! 0 + caseCmd@(T_CaseExpression _ var _) <- mapMaybe findCase body !!! 0 + + -- Make sure getopts name and case variable matches + [T_DollarBraced _ _ bracedWord] <- return $ getWordParts var + [T_Literal _ caseVar] <- return $ getWordParts bracedWord + guard $ caseVar == getoptsVar + return $ check (getId arg1) (map (:[]) $ filter (/= ':') options) caseCmd f _ = return () From c3aaa27540e1ab0e93074937176d8bbe89554afb Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 9 Oct 2021 12:13:41 -0700 Subject: [PATCH 506/763] Skip SC2214 if variable is modified in loop (fixes #2351) --- src/ShellCheck/Analytics.hs | 10 +--------- src/ShellCheck/AnalyzerLib.hs | 17 +++++++++++++++++ src/ShellCheck/Checks/Commands.hs | 5 +++++ 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 787a501..912fab5 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1957,17 +1957,9 @@ subshellAssignmentCheck params t = findSubshelled [] _ _ = return () findSubshelled (Assignment x@(_, _, str, data_):rest) scopes@((reason,scope):restscope) deadVars = - if isTrueAssignment data_ + if isTrueAssignmentSource data_ then findSubshelled rest ((reason, x:scope):restscope) $ Map.insert str Alive deadVars else findSubshelled rest scopes deadVars - where - isTrueAssignment c = - case c of - DataString SourceChecked -> False - DataString SourceDeclaration -> False - DataArray SourceChecked -> False - DataArray SourceDeclaration -> False - _ -> True findSubshelled (Reference (_, readToken, str):rest) scopes deadVars = do unless (shouldIgnore str) $ case Map.findWithDefault Alive str deadVars of diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index be54fbb..e36a14f 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -1005,6 +1005,23 @@ isUnmodifiedParameterExpansion t = in getBracedReference str == str _ -> False +isTrueAssignmentSource c = + case c of + DataString SourceChecked -> False + DataString SourceDeclaration -> False + DataArray SourceChecked -> False + DataArray SourceDeclaration -> False + _ -> True + +modifiesVariable params token name = + or $ map check flow + where + flow = getVariableFlow params token + check t = + case t of + Assignment (_, _, n, source) -> isTrueAssignmentSource source && n == name + _ -> False + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index edc41ed..e80ed3a 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -961,11 +961,13 @@ prop_checkWhileGetoptsCase4 = verifyNot checkWhileGetoptsCase "while getopts 'a: prop_checkWhileGetoptsCase5 = verifyNot checkWhileGetoptsCase "while getopts 'a:' x; do case $x in a) foo;; \\?) bar;; *) baz;; esac; done" prop_checkWhileGetoptsCase6 = verifyNot checkWhileGetoptsCase "while getopts 'a:b' x; do case $y in a) foo;; esac; done" prop_checkWhileGetoptsCase7 = verifyNot checkWhileGetoptsCase "while getopts 'a:b' x; do case x$x in xa) foo;; xb) foo;; esac; done" +prop_checkWhileGetoptsCase8 = verifyNot checkWhileGetoptsCase "while getopts 'a:b' x; do x=a; case $x in a) foo;; esac; done" checkWhileGetoptsCase = CommandCheck (Exactly "getopts") f where f :: Token -> Analysis f t@(T_SimpleCommand _ _ (cmd:arg1:name:_)) = do path <- getPathM t + params <- ask sequence_ $ do options <- getLiteralString arg1 getoptsVar <- getLiteralString name @@ -977,6 +979,9 @@ checkWhileGetoptsCase = CommandCheck (Exactly "getopts") f [T_Literal _ caseVar] <- return $ getWordParts bracedWord guard $ caseVar == getoptsVar + -- Make sure the variable isn't modified + guard . not $ modifiesVariable params (T_BraceGroup (Id 0) body) getoptsVar + return $ check (getId arg1) (map (:[]) $ filter (/= ':') options) caseCmd f _ = return () From 0d128dd918be22b764d1625393775eb4080ca006 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Fri, 15 Oct 2021 12:06:33 -0700 Subject: [PATCH 507/763] Mention known incompatibilities in man page --- shellcheck.1.md | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/shellcheck.1.md b/shellcheck.1.md index 1103acc..b1f0c25 100644 --- a/shellcheck.1.md +++ b/shellcheck.1.md @@ -335,10 +335,32 @@ locales where encoding is unspecified (such as the `C` locale). Windows users seeing `commitBuffer: invalid argument (invalid character)` should set their terminal to use UTF-8 with `chcp 65001`. -# AUTHORS +# KNOWN INCOMPATIBILITIES -ShellCheck is developed and maintained by Vidar Holen, with assistance from a -long list of wonderful contributors. +(If nothing in this section makes sense, you are unlikely to be affected by it) + +To avoid confusing and misguided suggestions, ShellCheck requires function +bodies to be either `{ brace groups; }` or `( subshells )`, and function names +containing `[]*=!` are only recognized after a `function` keyword. + +The following unconventional function definitions are identical in Bash, +but ShellCheck only recognizes the latter. + + [x!=y] () [[ $1 ]] + function [x!=y] () { [[ $1 ]]; } + +Shells without the `function` keyword do not allow these characters in function +names to begin with. Function names containing `{}` are not supported at all. + +Further, if ShellCheck sees `[x!=y]` it will assume this is an invalid +comparison. To invoke the above function, quote the command as in `'[x!=y]'`, +or to retain the same globbing behavior, use `command [x!=y]`. + +ShellCheck imposes additional restrictions on the `[` command to help diagnose +common invalid uses. While `[ $x= 1 ]` is defined in POSIX, ShellCheck will +assume it was intended as the much more likely comparison `[ "$x" = 1 ]` and +fail accordingly. For unconventional or dynamic uses of the `[` command, use +`test` or `\[` instead. # REPORTING BUGS @@ -346,6 +368,11 @@ Bugs and issues can be reported on GitHub: https://github.com/koalaman/shellcheck/issues +# AUTHORS + +ShellCheck is developed and maintained by Vidar Holen, with assistance from a +long list of wonderful contributors. + # COPYRIGHT Copyright 2012-2019, Vidar Holen and contributors. From 788aee1b7c1c74863c85837fbd53dd45cf810d14 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Fri, 15 Oct 2021 14:39:30 -0700 Subject: [PATCH 508/763] Treat typeset similar to declare (fixes #2354) --- src/ShellCheck/Analytics.hs | 1 + src/ShellCheck/AnalyzerLib.hs | 21 ++++++++++++++------- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 912fab5..8960d93 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2380,6 +2380,7 @@ prop_checkUnused46= verifyTree checkUnusedAssignments "readonly foo=(bar)" prop_checkUnused47= verifyNotTree checkUnusedAssignments "a=1; alias hello='echo $a'" prop_checkUnused48= verifyNotTree checkUnusedAssignments "_a=1" prop_checkUnused49= verifyNotTree checkUnusedAssignments "declare -A array; key=a; [[ -v array[$key] ]]" +prop_checkUnused50= verifyNotTree checkUnusedAssignments "foofunc() { :; }; typeset -fx foofunc" checkUnusedAssignments params t = execWriter (mapM_ warnFor unused) where diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index e36a14f..311364f 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -571,14 +571,12 @@ isClosingFileOp op = -- Consider 'export/declare -x' a reference, since it makes the var available getReferencedVariableCommand base@(T_SimpleCommand _ _ (T_NormalWord _ (T_Literal _ x:_):rest)) = case x of + "declare" -> forDeclare + "typeset" -> forDeclare + "export" -> if "f" `elem` flags then [] else concatMap getReference rest - "declare" -> if - any (`elem` flags) ["x", "p"] && - (not $ any (`elem` flags) ["f", "F"]) - then concatMap getReference rest - else [] "local" -> if "x" `elem` flags then concatMap getReference rest else [] @@ -589,6 +587,13 @@ getReferencedVariableCommand base@(T_SimpleCommand _ _ (T_NormalWord _ (T_Litera "alias" -> [(base, token, name) | token <- rest, name <- getVariablesFromLiteralToken token] _ -> [] where + forDeclare = + if + any (`elem` flags) ["x", "p"] && + (not $ any (`elem` flags) ["f", "F"]) + then concatMap getReference rest + else [] + getReference t@(T_Assignment _ _ name _ value) = [(t, t, name)] getReference t@(T_NormalWord _ [T_Literal _ name]) | not ("-" `isPrefixOf` name) = [(t, t, name)] getReference _ = [] @@ -628,8 +633,8 @@ getModifiedVariableCommand base@(T_SimpleCommand id cmdPrefix (T_NormalWord _ (T "export" -> if "f" `elem` flags then [] else concatMap getModifierParamString rest - "declare" -> if any (`elem` flags) ["F", "f", "p"] then [] else declaredVars - "typeset" -> declaredVars + "declare" -> forDeclare + "typeset" -> forDeclare "local" -> concatMap getModifierParamString rest "readonly" -> @@ -661,6 +666,8 @@ getModifiedVariableCommand base@(T_SimpleCommand id cmdPrefix (T_NormalWord _ (T T_NormalWord id1 [T_DoubleQuoted id2 [T_Literal id3 (stripEquals s)]] stripEqualsFrom t = t + forDeclare = if any (`elem` flags) ["F", "f", "p"] then [] else declaredVars + declaredVars = concatMap (getModifierParam defaultType) rest where defaultType = if any (`elem` flags) ["a", "A"] then DataArray else DataString From 7b2092b3cd89c98067e31cd68d2738f8afa2b0ad Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Fri, 15 Oct 2021 15:29:52 -0700 Subject: [PATCH 509/763] Give more examples of what ShellCheck looks for --- README.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d91f962..6f3e4a9 100644 --- a/README.md +++ b/README.md @@ -358,6 +358,7 @@ echo 'Don't forget to restart!' # Singlequote closed by apostrophe echo 'Don\'t try this at home' # Attempting to escape ' in '' echo 'Path is $PATH' # Variables in single quotes trap "echo Took ${SECONDS}s" 0 # Prematurely expanded trap +unset var[i] # Array index treated as glob ``` ### Conditionals @@ -376,6 +377,7 @@ ShellCheck can recognize many types of incorrect test statements. [ grep -q foo file ] # Command without $(..) [[ "$$file" == *.jpg ]] # Comparisons that can't succeed (( 1 -lt 2 )) # Using test operators in ((..)) +[ x ] & [ y ] | [ z ] # Accidental backgrounding and piping ``` ### Frequently misused commands @@ -447,6 +449,8 @@ echo "Hello $name" # Unassigned lowercase variables cmd | read bar; echo $bar # Assignments in subshells cat foo | cp bar # Piping to commands that don't read printf '%s: %s\n' foo # Mismatches in printf argument count +eval "${array[@]}" # Lost word boundaries in array eval +for i in "${x[@]}"; do ${x[$i]} # Using array value as key ``` ### Robustness @@ -471,6 +475,7 @@ ShellCheck will warn when using features not supported by the shebang. For examp echo {1..$n} # Works in ksh, but not bash/dash/sh echo {1..10} # Works in ksh and bash, but not dash/sh echo -n 42 # Works in ksh, bash and dash, undefined in sh +expr match str regex # Unportable alias for `expr str : regex` trap 'exit 42' sigint # Unportable signal spec cmd &> file # Unportable redirection operator read foo < /dev/tcp/host/22 # Unportable intercepted files @@ -491,10 +496,15 @@ rm “file” # Unicode quotes echo "Hello world" # Carriage return / DOS line endings echo hello \ # Trailing spaces after \ var=42 echo $var # Expansion of inlined environment -#!/bin/bash -x -e # Common shebang errors +!# bin/bash -x -e # Common shebang errors echo $((n/180*100)) # Unnecessary loss of precision ls *[:digit:].txt # Bad character class globs sed 's/foo/bar/' file > file # Redirecting to input +var2=$var2 # Variable assigned to itself +[ x$var = xval ] # Antiquated x-comparisons +ls() { ls -l "$@"; } # Infinitely recursive wrapper +alias ls='ls -l'; ls foo # Alias used before it takes effect +for x; do for x; do # Nested loop uses same variable while getopts "a" f; do case $f in "b") # Unhandled getopts flags ``` From 290fc8b945243bda1e867694c5abd5ecfdf2bc18 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Fri, 15 Oct 2021 18:03:05 -0700 Subject: [PATCH 510/763] Have quickscripts search for relevant paths (fixes #2286) --- quickrun | 10 +++++++++- quicktest | 11 ++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/quickrun b/quickrun index 172ae88..e0e0547 100755 --- a/quickrun +++ b/quickrun @@ -2,4 +2,12 @@ # quickrun runs ShellCheck in an interpreted mode. # This allows testing changes without recompiling. -runghc -isrc -idist/build/autogen shellcheck.hs "$@" +path=$(find . -type f -path './dist*/Paths_ShellCheck.hs' | sort | head -n 1) +if [ -z "$path" ] +then + echo >&2 "Unable to find Paths_ShellCheck.hs. Please 'cabal build' once." + exit 1 +fi +path="${path%/*}" + +exec runghc -isrc -i"$path" shellcheck.hs "$@" diff --git a/quicktest b/quicktest index 55041a7..6a1cf61 100755 --- a/quicktest +++ b/quicktest @@ -3,8 +3,17 @@ # This allows running tests without compiling, which can be faster. # 'cabal test' remains the source of truth. +path=$(find . -type f -path './dist*/Paths_ShellCheck.hs' | sort | head -n 1) +if [ -z "$path" ] +then + echo >&2 "Unable to find Paths_ShellCheck.hs. Please 'cabal build' once." + exit 1 +fi +path="${path%/*}" + + ( -var=$(echo 'main' | ghci test/shellcheck.hs 2>&1 | tee /dev/stderr) +var=$(echo 'main' | ghci -isrc -i"$path" test/shellcheck.hs 2>&1 | tee /dev/stderr) if [[ $var == *ExitSuccess* ]] then exit 0 From 0dd5c67bdf0fe5f0d03e4d2f6fc04c3796408ccd Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 21 Oct 2021 20:58:14 -0700 Subject: [PATCH 511/763] Warn about [^..] in Dash (fixes #2361) --- src/ShellCheck/Checks/ShellSupport.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 1d373e5..092c2bf 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -180,7 +180,7 @@ prop_checkBashisms95 = verify checkBashisms "#!/bin/sh\necho $_" prop_checkBashisms96 = verifyNot checkBashisms "#!/bin/dash\necho $_" prop_checkBashisms97 = verify checkBashisms "#!/bin/sh\necho ${var,}" prop_checkBashisms98 = verify checkBashisms "#!/bin/sh\necho ${var^^}" -prop_checkBashisms99 = verifyNot checkBashisms "#!/bin/dash\necho [^f]oo" +prop_checkBashisms99 = verify checkBashisms "#!/bin/dash\necho [^f]oo" checkBashisms = ForShell [Sh, Dash] $ \t -> do params <- ask kludge params t @@ -235,7 +235,7 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do where file = onlyLiteralString word isNetworked = any (`isPrefixOf` file) ["/dev/tcp", "/dev/udp"] - bashism (T_Glob id str) | not isDash && "[^" `isInfixOf` str = + bashism (T_Glob id str) | "[^" `isInfixOf` str = warnMsg id 3026 "^ in place of ! in glob bracket expressions is" bashism t@(TA_Variable id str _) | isBashVariable str = From efd49e486f47f7e10f9b01a8afd8f116eb0893c5 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 30 Oct 2021 17:47:30 -0700 Subject: [PATCH 512/763] Consider all forms of TA_Assignment to remove spaces (fixes #2364) --- src/ShellCheck/Analytics.hs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 8960d93..64ba8a8 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2048,6 +2048,7 @@ prop_checkSpacefulness42= verifyNotTree checkSpacefulness "run $1 --flags" prop_checkSpacefulness43= verifyNotTree checkSpacefulness "$foo=42" prop_checkSpacefulness44= verifyTree checkSpacefulness "#!/bin/sh\nexport var=$value" prop_checkSpacefulness45= verifyNotTree checkSpacefulness "wait -zzx -p foo; echo $foo" +prop_checkSpacefulness46= verifyNotTree checkSpacefulness "x=0; (( x += 1 )); echo $x" data SpaceStatus = SpaceSome | SpaceNone | SpaceEmpty deriving (Eq) instance Semigroup SpaceStatus where @@ -2138,6 +2139,7 @@ checkSpacefulness' onFind params t = where emit x = tell [x] + writeF _ (TA_Assignment {}) name _ = setSpaces name SpaceNone >> return [] writeF _ _ name (DataString SourceExternal) = setSpaces name SpaceSome >> return [] writeF _ _ name (DataString SourceInteger) = setSpaces name SpaceNone >> return [] From 8db220ae43280860f8ec920eea1e66608803d0f9 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 6 Nov 2021 15:37:59 -0700 Subject: [PATCH 513/763] Include `local -r` in check-extra-masked-returns (fixes #2362) --- src/ShellCheck/Analytics.hs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 64ba8a8..244236f 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -4782,6 +4782,8 @@ prop_checkExtraMaskedReturns31 = verifyNotTree checkExtraMaskedReturns "false < prop_checkExtraMaskedReturns32 = verifyNotTree checkExtraMaskedReturns "false < <(basename \"${BASH_SOURCE[0]}\")" prop_checkExtraMaskedReturns33 = verifyNotTree checkExtraMaskedReturns "{ false || true; } | true" prop_checkExtraMaskedReturns34 = verifyNotTree checkExtraMaskedReturns "{ false || :; } | true" +prop_checkExtraMaskedReturns35 = verifyTree checkExtraMaskedReturns "f() { local -r x=$(false); }" + checkExtraMaskedReturns params t = runNodeAnalysis findMaskingNodes params t where findMaskingNodes _ (T_Arithmetic _ list) = findMaskedNodesInList [list] @@ -4828,8 +4830,13 @@ checkExtraMaskedReturns params t = runNodeAnalysis findMaskingNodes params t isCheckedElsewhere t = hasParent isDeclaringCommand t where isDeclaringCommand t _ = fromMaybe False $ do - basename <- getCommandBasename t - return $ basename `elem` declaringCommands + cmd <- getCommand t + basename <- getCommandBasename cmd + return $ + case basename of + -- local -r x=$(false) is intentionally ignored for SC2155 + "local" | "r" `elem` (map snd $ getAllFlags cmd) -> False + _ -> basename `elem` declaringCommands isHarmlessCommand t = fromMaybe False $ do basename <- getCommandBasename t From bcca66eb6b7c18c58ac668d7ab47bb1890b489e3 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 6 Nov 2021 15:46:19 -0700 Subject: [PATCH 514/763] Update release checklist --- test/check_release | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/check_release b/test/check_release index 91001fb..fd1dbca 100755 --- a/test/check_release +++ b/test/check_release @@ -56,19 +56,19 @@ cat << EOF Manual Checklist $((i++)). Make sure none of the automated checks above failed -$((i++)). Make sure Travis build currently passes: https://travis-ci.org/koalaman/shellcheck +$((i++)). Make sure GitHub Build currently passes: https://github.com/koalaman/shellcheck/actions $((i++)). Make sure SnapCraft build currently works: https://build.snapcraft.io/user/koalaman $((i++)). Run test/distrotest to ensure that most distros can build OOTB. $((i++)). Format and read over the manual for bad formatting and outdated info. -$((i++)). Make sure the Hackage package builds, so that all files are +$((i++)). Make sure the Hackage package builds. Release Steps $((j++)). \`cabal sdist\` to generate a Hackage package $((j++)). \`git push --follow-tags\` to push commit -$((j++)). Wait for Travis to build +$((j++)). Wait for GitHub Actions to build. $((j++)). Verify release: - a. Check that the new versions are uploaded: https://shellcheck.storage.googleapis.com/index.html + a. Check that the new versions are uploaded: https://github.com/koalaman/shellcheck/tags b. Check that the docker images have version tags: https://hub.docker.com/u/koalaman $((j++)). If no disaster, upload to Hackage: http://hackage.haskell.org/upload $((j++)). Push a new commit that updates CHANGELOG.md From 71f1db660937d4d83e2709079eea444137bef4c3 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 6 Nov 2021 18:21:11 -0700 Subject: [PATCH 515/763] Update distro tests --- test/distrotest | 10 +++++----- test/stacktest | 5 +++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/test/distrotest b/test/distrotest index 346a706..50a5a17 100755 --- a/test/distrotest +++ b/test/distrotest @@ -64,13 +64,13 @@ ubuntu:latest apt-get update && apt-get install -y cabal-install haskell:latest true opensuse/leap:latest zypper install -y cabal-install ghc fedora:latest dnf install -y cabal-install ghc-template-haskell-devel findutils -archlinux/base:latest pacman -S -y --noconfirm cabal-install ghc-static base-devel +archlinux:latest pacman -S -y --noconfirm cabal-install ghc-static base-devel -# Other versions we want to support -ubuntu:18.04 apt-get update && apt-get install -y cabal-install +# Ubuntu LTS +ubuntu:20.04 apt-get update && apt-get install -y cabal-install -# Misc Haskell including current and latest Stack build -ubuntu:18.04 set -e; apt-get update && apt-get install -y curl && curl -sSL https://get.haskellstack.org/ | sh -s - -f && cd /mnt && exec test/stacktest +# Stack on Ubuntu LTS +ubuntu:20.04 set -e; apt-get update && apt-get install -y curl && curl -sSL https://get.haskellstack.org/ | sh -s - -f && cd /mnt && exec test/stacktest EOF exit "$final" diff --git a/test/stacktest b/test/stacktest index dc0113f..ae04f1b 100755 --- a/test/stacktest +++ b/test/stacktest @@ -18,10 +18,11 @@ command -v stack || stack setup || die "Failed to setup with default resolver" stack build --test || die "Failed to build/test with default resolver" +# Nice to haves, but not necessary for resolver in "${resolvers[@]}" do - stack --resolver="$resolver" setup || die "Failed to setup $resolver" - stack --resolver="$resolver" build --test || die "Failed build/test with $resolver!" + stack --resolver="$resolver" setup || die "Failed to setup $resolver. This probably doesn't matter." + stack --resolver="$resolver" build --test || die "Failed build/test with $resolver! This probably doesn't matter." done echo "Success" From 14a38b94cc2ca6810492a457d64f7019b059eed6 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 6 Nov 2021 18:59:24 -0700 Subject: [PATCH 516/763] Update stack resolver --- stack.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stack.yaml b/stack.yaml index 6dee632..4cf5c74 100644 --- a/stack.yaml +++ b/stack.yaml @@ -2,7 +2,7 @@ # For more information, see: https://docs.haskellstack.org/en/stable/yaml_configuration/ # Specifies the GHC version and set of packages available (e.g., lts-3.5, nightly-2015-09-21, ghc-7.10.2) -resolver: lts-13.26 +resolver: lts-18.15 # Local packages, usually specified by relative directory name packages: From 3b6972fbf14d98dded7ebfc65af6b73724be4efa Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 6 Nov 2021 19:07:34 -0700 Subject: [PATCH 517/763] Update copyright years --- shellcheck.1.md | 2 +- src/ShellCheck/ASTLib.hs | 2 +- src/ShellCheck/Analytics.hs | 2 +- src/ShellCheck/AnalyzerLib.hs | 2 +- src/ShellCheck/Checker.hs | 2 +- src/ShellCheck/Checks/Commands.hs | 2 +- src/ShellCheck/Checks/ShellSupport.hs | 2 +- src/ShellCheck/Parser.hs | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/shellcheck.1.md b/shellcheck.1.md index b1f0c25..146d791 100644 --- a/shellcheck.1.md +++ b/shellcheck.1.md @@ -375,7 +375,7 @@ long list of wonderful contributors. # COPYRIGHT -Copyright 2012-2019, Vidar Holen and contributors. +Copyright 2012-2021, Vidar Holen and contributors. Licensed under the GNU General Public License version 3 or later, see https://gnu.org/licenses/gpl.html diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 5c13697..83ba5f8 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -1,5 +1,5 @@ {- - Copyright 2012-2019 Vidar Holen + Copyright 2012-2021 Vidar Holen This file is part of ShellCheck. https://www.shellcheck.net diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 244236f..652c2fb 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1,5 +1,5 @@ {- - Copyright 2012-2019 Vidar Holen + Copyright 2012-2021 Vidar Holen This file is part of ShellCheck. https://www.shellcheck.net diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 311364f..687859f 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -1,5 +1,5 @@ {- - Copyright 2012-2019 Vidar Holen + Copyright 2012-2021 Vidar Holen This file is part of ShellCheck. https://www.shellcheck.net diff --git a/src/ShellCheck/Checker.hs b/src/ShellCheck/Checker.hs index cce1063..ef8182f 100644 --- a/src/ShellCheck/Checker.hs +++ b/src/ShellCheck/Checker.hs @@ -1,5 +1,5 @@ {- - Copyright 2012-2019 Vidar Holen + Copyright 2012-2020 Vidar Holen This file is part of ShellCheck. https://www.shellcheck.net diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index e80ed3a..5a29a26 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -1,5 +1,5 @@ {- - Copyright 2012-2019 Vidar Holen + Copyright 2012-2021 Vidar Holen This file is part of ShellCheck. https://www.shellcheck.net diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 092c2bf..22a6a5f 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -1,5 +1,5 @@ {- - Copyright 2012-2016 Vidar Holen + Copyright 2012-2020 Vidar Holen This file is part of ShellCheck. https://www.shellcheck.net diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index e269b4d..92eb61f 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -1,5 +1,5 @@ {- - Copyright 2012-2019 Vidar Holen + Copyright 2012-2021 Vidar Holen This file is part of ShellCheck. https://www.shellcheck.net From eea823e3d088c0bcf048dae9ad9c321be09560ad Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 6 Nov 2021 22:05:19 -0700 Subject: [PATCH 518/763] Fix bad version on stable releases --- .github/workflows/build.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e28a43f..5595219 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,22 +19,22 @@ jobs: with: fetch-depth: 0 - - name: Package Source - run: | - ./setgitversion - mkdir source - cabal sdist - mv dist-newstyle/sdist/*.tar.gz source/source.tar.gz - - name: Deduce tags run: | - exec > source/tags - echo "latest" + mkdir source + echo "latest" > source/tags if tag=$(git describe --exact-match --tags) then - echo "stable" - echo "$tag" + echo "stable" >> source/tags + echo "$tag" >> source/tags fi + cat source/tags + + - name: Package Source + run: | + grep "stable" source/tags || ./setgitversion + cabal sdist + mv dist-newstyle/sdist/*.tar.gz source/source.tar.gz - name: Upload artifact uses: actions/upload-artifact@v2 From e5ad4cf420a7f7b8e5eaac872b14a1619051cf10 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 6 Nov 2021 19:08:58 -0700 Subject: [PATCH 519/763] Stable version 0.8.0 This release is dedicated to dibblego, who pushed me down the Haskell rabbit hole. In 2006 I thought you were crazy. Today I *know* you are. --- CHANGELOG.md | 2 +- ShellCheck.cabal | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a1456a0..9118671 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## Git (0.8.0) +## v0.8.0 - 2021-11-06 ### Added - `disable=all` now conveniently disables all warnings - `external-sources=true` directive can be added to .shellcheckrc to make diff --git a/ShellCheck.cabal b/ShellCheck.cabal index 9433f55..1167c82 100644 --- a/ShellCheck.cabal +++ b/ShellCheck.cabal @@ -1,5 +1,5 @@ Name: ShellCheck -Version: 0.7.2 +Version: 0.8.0 Synopsis: Shell script analysis tool License: GPL-3 License-file: LICENSE From 4c186c20b9a5a3a95768d9a609c93df37991c663 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 6 Nov 2021 23:18:19 -0700 Subject: [PATCH 520/763] Post-release CHANGELOG update --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9118671..f65bbfa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +## Git +### Added + +### Fixed + +### Changed + + ## v0.8.0 - 2021-11-06 ### Added - `disable=all` now conveniently disables all warnings From c5de58ae84954b1c773bfe2b3710daf0d7e92fbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Skytt=C3=A4?= Date: Sat, 13 Nov 2021 12:50:53 +0200 Subject: [PATCH 521/763] Comment spelling fixes --- src/ShellCheck/AnalyzerLib.hs | 2 +- src/ShellCheck/Checks/Commands.hs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 687859f..5f77fba 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -351,7 +351,7 @@ isQuoteFreeNode strict shell tree t = T_SelectIn {} -> return (not strict) _ -> Nothing - -- Check whether this assigment is self-quoting due to being a recognized + -- Check whether this assignment is self-quoting due to being a recognized -- assignment passed to a Declaration Utility. This will soon be required -- by POSIX: https://austingroupbugs.net/view.php?id=351 assignmentIsQuoting t = shellParsesParamsAsAssignments || not (isAssignmentParamToCommand t) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 5a29a26..1a48a28 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -735,7 +735,7 @@ getPrintfFormats = getFormats -- \____ _____/\___ ____/ \____ ____/\_________ _________/ \______ / -- V V V V V -- flags field width precision format character rest - -- field width and precision can be specified with a '*' instead of a digit, + -- field width and precision can be specified with an '*' instead of a digit, -- in which case printf will accept one more argument for each '*' used From d9a9d5db86122d608b1a6929609e799d175e6aa6 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 14 Nov 2021 16:39:32 -0800 Subject: [PATCH 522/763] Mark prefix/postfix inc/dec as integers (fixes #2376) --- src/ShellCheck/Analytics.hs | 3 ++- src/ShellCheck/AnalyzerLib.hs | 8 +++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 652c2fb..188d683 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2049,6 +2049,8 @@ prop_checkSpacefulness43= verifyNotTree checkSpacefulness "$foo=42" prop_checkSpacefulness44= verifyTree checkSpacefulness "#!/bin/sh\nexport var=$value" prop_checkSpacefulness45= verifyNotTree checkSpacefulness "wait -zzx -p foo; echo $foo" prop_checkSpacefulness46= verifyNotTree checkSpacefulness "x=0; (( x += 1 )); echo $x" +prop_checkSpacefulness47= verifyNotTree checkSpacefulness "x=0; (( x-- )); echo $x" +prop_checkSpacefulness48= verifyNotTree checkSpacefulness "x=0; (( ++x )); echo $x" data SpaceStatus = SpaceSome | SpaceNone | SpaceEmpty deriving (Eq) instance Semigroup SpaceStatus where @@ -2139,7 +2141,6 @@ checkSpacefulness' onFind params t = where emit x = tell [x] - writeF _ (TA_Assignment {}) name _ = setSpaces name SpaceNone >> return [] writeF _ _ name (DataString SourceExternal) = setSpaces name SpaceSome >> return [] writeF _ _ name (DataString SourceInteger) = setSpaces name SpaceNone >> return [] diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 687859f..9b53f9f 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -511,13 +511,11 @@ getModifiedVariables t = T_SimpleCommand {} -> getModifiedVariableCommand t - TA_Unary _ "++|" v@(TA_Variable _ name _) -> - [(t, v, name, DataString $ SourceFrom [v])] - TA_Unary _ "|++" v@(TA_Variable _ name _) -> - [(t, v, name, DataString $ SourceFrom [v])] + TA_Unary _ op v@(TA_Variable _ name _) | "--" `isInfixOf` op || "++" `isInfixOf` op -> + [(t, v, name, DataString SourceInteger)] TA_Assignment _ op (TA_Variable _ name _) rhs -> do guard $ op `elem` ["=", "*=", "/=", "%=", "+=", "-=", "<<=", ">>=", "&=", "^=", "|="] - return (t, t, name, DataString $ SourceFrom [rhs]) + return (t, t, name, DataString SourceInteger) T_BatsTest {} -> [ (t, t, "lines", DataArray SourceExternal), From 499c99372eaef411fc5223393c6799005eebc085 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 14 Nov 2021 21:34:21 -0800 Subject: [PATCH 523/763] Rewrite SC2032 warning and mention line number (fixes #2353) --- src/ShellCheck/Analytics.hs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 188d683..4a7f364 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2297,7 +2297,7 @@ checkFunctionsUsedExternally params t = let args = skipOver t argv let argStrings = map (\x -> (fromMaybe "" $ getLiteralString x, x)) args let candidates = getPotentialCommands name argStrings - mapM_ (checkArg name) candidates + mapM_ (checkArg name (getId t)) candidates _ -> return () checkCommand _ _ = return () @@ -2323,14 +2323,19 @@ checkFunctionsUsedExternally params t = functionsAndAliases = Map.union (functions t) (aliases t) - checkArg cmd (_, arg) = sequence_ $ do + patternContext id = + case posLine . fst <$> Map.lookup id (tokenPositions params) of + Just l -> " on line " <> show l <> "." + _ -> "." + + checkArg cmd cmdId (_, arg) = sequence_ $ do literalArg <- getUnquotedLiteral arg -- only consider unquoted literals definitionId <- Map.lookup literalArg functionsAndAliases return $ do warn (getId arg) 2033 - "Shell functions can't be passed to external commands." + "Shell functions can't be passed to external commands. Use separate script or sh -c." info definitionId 2032 $ - "Use own script or sh -c '..' to run this from " ++ cmd ++ "." + "This function can't be invoked via " ++ cmd ++ patternContext cmdId prop_checkUnused0 = verifyNotTree checkUnusedAssignments "var=foo; echo $var" prop_checkUnused1 = verifyTree checkUnusedAssignments "var=foo; echo $bar" From 9092080a84af694f1a1fbc1d59f89d296f362069 Mon Sep 17 00:00:00 2001 From: Martin Schulze Date: Mon, 15 Nov 2021 11:49:36 +0100 Subject: [PATCH 524/763] bats: Add check for useless negation (SC2314/15) --- src/ShellCheck/Analytics.hs | 43 +++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 4a7f364..7c31137 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -199,6 +199,7 @@ nodeChecks = [ ,checkComparisonWithLeadingX ,checkCommandWithTrailingSymbol ,checkUnquotedParameterExpansionPattern + ,checkBatsTestDoesNotUseNegation ] optionalChecks = map fst optionalTreeChecks @@ -4863,5 +4864,47 @@ checkExtraMaskedReturns params t = runNodeAnalysis findMaskingNodes params t hasParent pred t = any (uncurry pred) (parentChildPairs t) +-- hard error on negated command that is not last +prop_checkBatsTestDoesNotUseNegation1 = verify checkBatsTestDoesNotUseNegation "#!/usr/bin/env/bats\n@test \"name\" { ! true; false; }" +prop_checkBatsTestDoesNotUseNegation2 = verify checkBatsTestDoesNotUseNegation "#!/usr/bin/env/bats\n@test \"name\" { ! [[ -e test ]]; false; }" +prop_checkBatsTestDoesNotUseNegation3 = verify checkBatsTestDoesNotUseNegation "#!/usr/bin/env/bats\n@test \"name\" { ! [ -e test ]; false; }" +-- acceptable formats: +-- using run +prop_checkBatsTestDoesNotUseNegation4 = verifyNot checkBatsTestDoesNotUseNegation "#!/usr/bin/env/bats\n@test \"name\" { run ! true; }" +-- using || false +prop_checkBatsTestDoesNotUseNegation5 = verifyNot checkBatsTestDoesNotUseNegation "#!/usr/bin/env/bats\n@test \"name\" { ! [[ -e test ]] || false; }" +prop_checkBatsTestDoesNotUseNegation6 = verifyNot checkBatsTestDoesNotUseNegation "#!/usr/bin/env/bats\n@test \"name\" { ! [ -e test ] || false; }" +-- only style warning when last command +prop_checkBatsTestDoesNotUseNegation7 = verifyCodes checkBatsTestDoesNotUseNegation [2314] "#!/usr/bin/env/bats\n@test \"name\" { ! true; }" +prop_checkBatsTestDoesNotUseNegation8 = verifyCodes checkBatsTestDoesNotUseNegation [2315] "#!/usr/bin/env/bats\n@test \"name\" { ! [[ -e test ]]; }" +prop_checkBatsTestDoesNotUseNegation9 = verifyCodes checkBatsTestDoesNotUseNegation [2315] "#!/usr/bin/env/bats\n@test \"name\" { ! [ -e test ]; }" + +checkBatsTestDoesNotUseNegation params t = + case t of + T_BatsTest _ _ (T_BraceGroup _ commands) -> mapM_ (check commands) commands + _ -> return () + where + check commands t = + case t of + T_Banged id (T_Pipeline _ _ [T_Redirecting _ _ (T_Condition idCondition _ _)]) -> + if t `isLastOf` commands + then style id 2315 "In Bats, ! will not fail the test if it is not the last command anymore. Fold the `!` into the conditional!" + else err id 2315 + "In Bats, ! does not cause a test failure. Fold the `!` into the conditional!" + + T_Banged id cmd -> if t `isLastOf` commands + then styleWithFix id 2314 + "In Bats, ! will not fail the test if it is not the last command anymore. Use `run ! ` (on Bats >= 1.5.0) instead." + (fixWith [replaceStart id params 0 "run "]) + else errWithFix id 2314 + "In Bats, ! does not cause a test failure. Use 'run ! ' (on Bats >= 1.5.0) instead." + (fixWith [replaceStart id params 0 "run "]) + _ -> return () + isLastOf t commands = + case commands of + [x] -> x == t + x:rest -> isLastOf t rest + [] -> False + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) From d7971dafd1539de37bb29cb554d5fd4f4b95553b Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 4 Dec 2021 17:37:12 -0800 Subject: [PATCH 525/763] Minor formatting fixes --- src/ShellCheck/Analytics.hs | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 7c31137..d48104e 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -4868,7 +4868,7 @@ checkExtraMaskedReturns params t = runNodeAnalysis findMaskingNodes params t prop_checkBatsTestDoesNotUseNegation1 = verify checkBatsTestDoesNotUseNegation "#!/usr/bin/env/bats\n@test \"name\" { ! true; false; }" prop_checkBatsTestDoesNotUseNegation2 = verify checkBatsTestDoesNotUseNegation "#!/usr/bin/env/bats\n@test \"name\" { ! [[ -e test ]]; false; }" prop_checkBatsTestDoesNotUseNegation3 = verify checkBatsTestDoesNotUseNegation "#!/usr/bin/env/bats\n@test \"name\" { ! [ -e test ]; false; }" --- acceptable formats: +-- acceptable formats: -- using run prop_checkBatsTestDoesNotUseNegation4 = verifyNot checkBatsTestDoesNotUseNegation "#!/usr/bin/env/bats\n@test \"name\" { run ! true; }" -- using || false @@ -4886,21 +4886,18 @@ checkBatsTestDoesNotUseNegation params t = where check commands t = case t of - T_Banged id (T_Pipeline _ _ [T_Redirecting _ _ (T_Condition idCondition _ _)]) -> + T_Banged id (T_Pipeline _ _ [T_Redirecting _ _ (T_Condition idCondition _ _)]) -> if t `isLastOf` commands then style id 2315 "In Bats, ! will not fail the test if it is not the last command anymore. Fold the `!` into the conditional!" - else err id 2315 - "In Bats, ! does not cause a test failure. Fold the `!` into the conditional!" - + else err id 2315 "In Bats, ! does not cause a test failure. Fold the `!` into the conditional!" + T_Banged id cmd -> if t `isLastOf` commands - then styleWithFix id 2314 - "In Bats, ! will not fail the test if it is not the last command anymore. Use `run ! ` (on Bats >= 1.5.0) instead." + then styleWithFix id 2314 "In Bats, ! will not fail the test if it is not the last command anymore. Use `run ! ` (on Bats >= 1.5.0) instead." (fixWith [replaceStart id params 0 "run "]) - else errWithFix id 2314 - "In Bats, ! does not cause a test failure. Use 'run ! ' (on Bats >= 1.5.0) instead." + else errWithFix id 2314 "In Bats, ! does not cause a test failure. Use 'run ! ' (on Bats >= 1.5.0) instead." (fixWith [replaceStart id params 0 "run "]) _ -> return () - isLastOf t commands = + isLastOf t commands = case commands of [x] -> x == t x:rest -> isLastOf t rest From 3a118246ef2e6e37129cb0f1c0d1dae319385520 Mon Sep 17 00:00:00 2001 From: Rune Juhl Jacobsen Date: Tue, 14 Dec 2021 16:00:47 +0100 Subject: [PATCH 526/763] Fix bug in 2126 when using after/before flags with grep Using `--after-context`/`-A` or `--before-context`/`-B` would give a warning recommending the user to use `grep -c`, even though that would give a different result than using `grep | wc -l`: ```fundamental $ echo -e "1\n2\n3" | grep -cA 3 1 1 $ echo -e "1\n2\n3" | grep -A 3 1 | wc -l 3 ``` --- src/ShellCheck/Analytics.hs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index d48104e..e6e47ea 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -545,6 +545,10 @@ prop_checkPipePitfalls15 = verifyNot checkPipePitfalls "foo | grep bar | wc -cmw prop_checkPipePitfalls16 = verifyNot checkPipePitfalls "foo | grep -r bar | wc -l" prop_checkPipePitfalls17 = verifyNot checkPipePitfalls "foo | grep -l bar | wc -l" prop_checkPipePitfalls18 = verifyNot checkPipePitfalls "foo | grep -L bar | wc -l" +prop_checkPipePitfalls19 = verifyNot checkPipePitfalls "foo | grep -A2 bar | wc -l" +prop_checkPipePitfalls20 = verifyNot checkPipePitfalls "foo | grep -B999 bar | wc -l" +prop_checkPipePitfalls21 = verifyNot checkPipePitfalls "foo | grep --after-context 999 bar | wc -l" +prop_checkPipePitfalls22 = verifyNot checkPipePitfalls "foo | grep -B 1 --after-context 999 bar | wc -l" checkPipePitfalls _ (T_Pipeline id _ commands) = do for ["find", "xargs"] $ \(find:xargs:_) -> @@ -566,10 +570,10 @@ checkPipePitfalls _ (T_Pipeline id _ commands) = do let flagsGrep = maybe [] (map snd . getAllFlags) $ getCommand grep flagsWc = maybe [] (map snd . getAllFlags) $ getCommand wc in - unless (any (`elem` ["l", "files-with-matches", "L", "files-without-matches", "o", "only-matching", "r", "R", "recursive"]) flagsGrep + unless (any (`elem` ["l", "files-with-matches", "L", "files-without-matches", "o", "only-matching", "r", "R", "recursive", "A", "after-context", "B", "before-context"]) flagsGrep || any (`elem` ["m", "chars", "w", "words", "c", "bytes", "L", "max-line-length"]) flagsWc || null flagsWc) $ - style (getId grep) 2126 "Consider using grep -c instead of grep|wc -l." + style (getId grep) 2126 "Consider using 'grep -c' instead of 'grep|wc -l'." didLs <- fmap or . sequence $ [ for' ["ls", "grep"] $ From e6e558946ca70985dfe1ac905af19e615cbc4cc6 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 21 Dec 2021 14:07:46 -0800 Subject: [PATCH 527/763] Improve decoding of single quoted literals (fixes #2418) --- src/ShellCheck/ASTLib.hs | 35 ++++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 83ba5f8..7c88432 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -369,6 +369,21 @@ getGlobOrLiteralString = getLiteralStringExt f f (T_Glob _ str) = return str f _ = Nothing + +prop_getLiteralString1 = getLiteralString (T_DollarSingleQuoted (Id 0) "\\x01") == Just "\1" +prop_getLiteralString2 = getLiteralString (T_DollarSingleQuoted (Id 0) "\\xyz") == Just "\\xyz" +prop_getLiteralString3 = getLiteralString (T_DollarSingleQuoted (Id 0) "\\x1") == Just "\x1" +prop_getLiteralString4 = getLiteralString (T_DollarSingleQuoted (Id 0) "\\x1y") == Just "\x1y" +prop_getLiteralString5 = getLiteralString (T_DollarSingleQuoted (Id 0) "\\xy") == Just "\\xy" +prop_getLiteralString6 = getLiteralString (T_DollarSingleQuoted (Id 0) "\\x") == Just "\\x" +prop_getLiteralString7 = getLiteralString (T_DollarSingleQuoted (Id 0) "\\1x") == Just "\1x" +prop_getLiteralString8 = getLiteralString (T_DollarSingleQuoted (Id 0) "\\12x") == Just "\o12x" +prop_getLiteralString9 = getLiteralString (T_DollarSingleQuoted (Id 0) "\\123x") == Just "\o123x" +prop_getLiteralString10 = getLiteralString (T_DollarSingleQuoted (Id 0) "\\1234") == Just "\o123\&4" +prop_getLiteralString11 = getLiteralString (T_DollarSingleQuoted (Id 0) "\\1") == Just "\1" +prop_getLiteralString12 = getLiteralString (T_DollarSingleQuoted (Id 0) "\\12") == Just "\o12" +prop_getLiteralString13 = getLiteralString (T_DollarSingleQuoted (Id 0) "\\123") == Just "\o123" + -- Maybe get the literal value of a token, using a custom function -- to map unrecognized Tokens into strings. getLiteralStringExt :: Monad m => (Token -> m String) -> Token -> m String @@ -401,14 +416,15 @@ getLiteralStringExt more = g '\\' -> '\\' : rest 'x' -> case cs of - (x:y:more) -> - if isHexDigit x && isHexDigit y - then chr (16*(digitToInt x) + (digitToInt y)) : rest - else '\\':c:rest + (x:y:more) | isHexDigit x && isHexDigit y -> + chr (16*(digitToInt x) + (digitToInt y)) : decodeEscapes more + (x:more) | isHexDigit x -> + chr (digitToInt x) : decodeEscapes more + more -> '\\' : 'x' : decodeEscapes more _ | isOctDigit c -> - let digits = take 3 $ takeWhile isOctDigit (c:cs) - num = parseOct digits - in (if num < 256 then chr num else '?') : rest + let (digits, more) = spanMax isOctDigit 3 (c:cs) + num = (parseOct digits) `mod` 256 + in (chr num) : decodeEscapes more _ -> '\\' : c : rest where rest = decodeEscapes cs @@ -416,6 +432,11 @@ getLiteralStringExt more = g where f n "" = n f n (c:rest) = f (n * 8 + digitToInt c) rest + spanMax f n list = + let (first, second) = span f list + (prefix, suffix) = splitAt n first + in + (prefix, suffix ++ second) decodeEscapes (c:cs) = c : decodeEscapes cs decodeEscapes [] = [] From ade2bf7b871362c9c30878294aef8ab775f3ea07 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 9 Jan 2022 16:50:50 -0800 Subject: [PATCH 528/763] Allow parsing [[ x = ["$y"] ]] (fixes #2165) --- src/ShellCheck/Parser.hs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 92eb61f..8fbdb5a 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -556,7 +556,7 @@ readConditionContents single = notFollowedBy2 (try (spacing >> string "]")) x <- readNormalWord pos <- getPosition - when (endedWith "]" x && notArrayIndex x) $ do + when (notArrayIndex x && endedWith "]" x && not (x `containsLiteral` "[")) $ do parseProblemAt pos ErrorC 1020 $ "You need a space before the " ++ (if single then "]" else "]]") ++ "." fail "Missing space before ]" @@ -572,6 +572,7 @@ readConditionContents single = endedWith _ _ = False notArrayIndex (T_NormalWord id s@(_:T_Literal _ t:_)) = t /= "[" notArrayIndex _ = True + containsLiteral x s = s `isInfixOf` onlyLiteralString x readCondAndOp = readAndOrOp TC_And "&&" False <|> readAndOrOp TC_And "-a" True @@ -941,6 +942,9 @@ prop_readCondition23 = isOk readCondition "[[ -v arr[$var] ]]" prop_readCondition25 = isOk readCondition "[[ lex.yy.c -ot program.l ]]" prop_readCondition26 = isOk readScript "[[ foo ]]\\\n && bar" prop_readCondition27 = not $ isOk readConditionCommand "[[ x ]] foo" +prop_readCondition28 = isOk readCondition "[[ x = [\"$1\"] ]]" +prop_readCondition29 = isOk readCondition "[[ x = [*] ]]" + readCondition = called "test expression" $ do opos <- getPosition start <- startSpan From 2292e852e5be3a727a667dce9def79c1d12f0804 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 23 Jan 2022 14:23:56 -0800 Subject: [PATCH 529/763] Switch linux-x86_64 build from Ubuntu to Alpine for musl --- build/linux.x86_64/Dockerfile | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/build/linux.x86_64/Dockerfile b/build/linux.x86_64/Dockerfile index f0ad16a..3112ac2 100644 --- a/build/linux.x86_64/Dockerfile +++ b/build/linux.x86_64/Dockerfile @@ -1,16 +1,10 @@ -FROM ubuntu:20.04 +FROM alpine:latest ENV TARGETNAME linux.x86_64 # Install GHC and cabal USER root -ENV DEBIAN_FRONTEND noninteractive -RUN apt-get update && apt-get install -y ghc curl xz-utils - -# So we'd like a later version of Cabal that supports --enable-executable-static, -# but we can't use Ubuntu 20.10 because coreutils has switched to new syscalls that -# the TravisCI kernel doesn't support. Download it manually. -RUN curl "https://downloads.haskell.org/~cabal/cabal-install-3.2.0.0/cabal-install-3.2.0.0-x86_64-unknown-linux.tar.xz" | tar xJv -C /usr/bin +RUN apk add ghc cabal g++ libffi-dev curl bash # Use ld.bfd instead of ld.gold due to # x86_64-linux-gnu/libpthread.a(pthread_cond_init.o)(.note.stapsdt+0x14): error: From 88cdb4e2c9b45becb21bd02cd7b205d5bef8cb56 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 3 Feb 2022 19:23:46 -0800 Subject: [PATCH 530/763] Warn about spaces around = in alias (fixes #2442) --- src/ShellCheck/Checks/Commands.hs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 1a48a28..72d8c09 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -98,7 +98,7 @@ commandChecks = [ ,checkUnquotedEchoSpaces ,checkEvalArray ] - ++ map checkArgComparison declaringCommands + ++ map checkArgComparison ("alias" : declaringCommands) ++ map checkMaskedReturns declaringCommands @@ -1253,6 +1253,7 @@ prop_checkArgComparison3 = verifyNot (checkArgComparison "declare") "declare a=b prop_checkArgComparison4 = verify (checkArgComparison "export") "export a +=b" prop_checkArgComparison7 = verifyNot (checkArgComparison "declare") "declare -a +i foo" prop_checkArgComparison8 = verify (checkArgComparison "let") "let x = 0" +prop_checkArgComparison9 = verify (checkArgComparison "alias") "alias x =0" -- This mirrors checkSecondArgIsComparison but for arguments to local/readonly/declare/export checkArgComparison cmd = CommandCheck (Exactly cmd) wordsWithEqual where From fa15c0a454be93603418a56ed0d625e5c76a83fc Mon Sep 17 00:00:00 2001 From: Patrick Xia Date: Thu, 5 May 2022 16:09:02 -0700 Subject: [PATCH 531/763] add SC2316: error on multiple declarations like 'readonly local' --- src/ShellCheck/Checks/Commands.hs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 72d8c09..bad4cf2 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -100,6 +100,7 @@ commandChecks = [ ] ++ map checkArgComparison ("alias" : declaringCommands) ++ map checkMaskedReturns declaringCommands + ++ map checkMultipleDeclaring declaringCommands optionalChecks = map fst optionalCommandChecks @@ -941,6 +942,21 @@ checkLocalScope = CommandCheck (Exactly "local") $ \t -> unless (any isFunctionLike path) $ err (getId $ getCommandTokenOrThis t) 2168 "'local' is only valid in functions." +prop_checkMultipleDeclaring1 = verify (checkMultipleDeclaring "local") "q() { local readonly var=1; }" +prop_checkMultipleDeclaring2 = verifyNot (checkMultipleDeclaring "local") "q() { local var=1; }" +prop_checkMultipleDeclaring3 = verify (checkMultipleDeclaring "readonly") "readonly local foo=5" +prop_checkMultipleDeclaring4 = verify (checkMultipleDeclaring "export") "export readonly foo=5" +prop_checkMultipleDeclaring5 = verifyNot (checkMultipleDeclaring "local") "f() { local -r foo=5; }" +prop_checkMultipleDeclaring6 = verifyNot (checkMultipleDeclaring "declare") "declare -rx foo=5" +checkMultipleDeclaring cmd = CommandCheck (Exactly cmd) (mapM_ check . arguments) + where + check t = sequence_ $ do + lit <- getLiteralString t + guard $ lit `elem` declaringCommands + return $ err (getId $ getCommandTokenOrThis t) 2316 $ + "This applies " ++ cmd ++ " to the variable named " ++ lit ++ + ", which is probably not what you want. Use a separate command or the appropriate `declare` options instead." + prop_checkDeprecatedTempfile1 = verify checkDeprecatedTempfile "var=$(tempfile)" prop_checkDeprecatedTempfile2 = verifyNot checkDeprecatedTempfile "tempfile=$(mktemp)" checkDeprecatedTempfile = CommandCheck (Basename "tempfile") $ From 282155268829ffe450a21eb728ba0417c9c2578d Mon Sep 17 00:00:00 2001 From: Rune Juhl Jacobsen Date: Tue, 14 Dec 2021 16:00:47 +0100 Subject: [PATCH 532/763] Fix bug in 2126 when using after/before flags with grep Using `--after-context`/`-A` or `--before-context`/`-B` would give a warning recommending the user to use `grep -c`, even though that would give a different result than using `grep | wc -l`: ```fundamental $ echo -e "1\n2\n3" | grep -cA 3 1 1 $ echo -e "1\n2\n3" | grep -A 3 1 | wc -l 3 ``` --- src/ShellCheck/Analytics.hs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index d48104e..e6e47ea 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -545,6 +545,10 @@ prop_checkPipePitfalls15 = verifyNot checkPipePitfalls "foo | grep bar | wc -cmw prop_checkPipePitfalls16 = verifyNot checkPipePitfalls "foo | grep -r bar | wc -l" prop_checkPipePitfalls17 = verifyNot checkPipePitfalls "foo | grep -l bar | wc -l" prop_checkPipePitfalls18 = verifyNot checkPipePitfalls "foo | grep -L bar | wc -l" +prop_checkPipePitfalls19 = verifyNot checkPipePitfalls "foo | grep -A2 bar | wc -l" +prop_checkPipePitfalls20 = verifyNot checkPipePitfalls "foo | grep -B999 bar | wc -l" +prop_checkPipePitfalls21 = verifyNot checkPipePitfalls "foo | grep --after-context 999 bar | wc -l" +prop_checkPipePitfalls22 = verifyNot checkPipePitfalls "foo | grep -B 1 --after-context 999 bar | wc -l" checkPipePitfalls _ (T_Pipeline id _ commands) = do for ["find", "xargs"] $ \(find:xargs:_) -> @@ -566,10 +570,10 @@ checkPipePitfalls _ (T_Pipeline id _ commands) = do let flagsGrep = maybe [] (map snd . getAllFlags) $ getCommand grep flagsWc = maybe [] (map snd . getAllFlags) $ getCommand wc in - unless (any (`elem` ["l", "files-with-matches", "L", "files-without-matches", "o", "only-matching", "r", "R", "recursive"]) flagsGrep + unless (any (`elem` ["l", "files-with-matches", "L", "files-without-matches", "o", "only-matching", "r", "R", "recursive", "A", "after-context", "B", "before-context"]) flagsGrep || any (`elem` ["m", "chars", "w", "words", "c", "bytes", "L", "max-line-length"]) flagsWc || null flagsWc) $ - style (getId grep) 2126 "Consider using grep -c instead of grep|wc -l." + style (getId grep) 2126 "Consider using 'grep -c' instead of 'grep|wc -l'." didLs <- fmap or . sequence $ [ for' ["ls", "grep"] $ From fd595d1058a7991a3e925922be2da96c3c023faa Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Fri, 6 May 2022 10:06:12 -0700 Subject: [PATCH 533/763] Only trigger SC2316 on unquoted words. --- src/ShellCheck/Checks/Commands.hs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index bad4cf2..e65dc68 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -948,10 +948,11 @@ prop_checkMultipleDeclaring3 = verify (checkMultipleDeclaring "readonly") "reado prop_checkMultipleDeclaring4 = verify (checkMultipleDeclaring "export") "export readonly foo=5" prop_checkMultipleDeclaring5 = verifyNot (checkMultipleDeclaring "local") "f() { local -r foo=5; }" prop_checkMultipleDeclaring6 = verifyNot (checkMultipleDeclaring "declare") "declare -rx foo=5" +prop_checkMultipleDeclaring7 = verifyNot (checkMultipleDeclaring "readonly") "readonly 'local' foo=5" checkMultipleDeclaring cmd = CommandCheck (Exactly cmd) (mapM_ check . arguments) where check t = sequence_ $ do - lit <- getLiteralString t + lit <- getUnquotedLiteral t guard $ lit `elem` declaringCommands return $ err (getId $ getCommandTokenOrThis t) 2316 $ "This applies " ++ cmd ++ " to the variable named " ++ lit ++ From 399c04cc17931c5221672257666f39eda6aefc14 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Fri, 6 May 2022 10:11:52 -0700 Subject: [PATCH 534/763] Mention SC2316 in changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f65bbfa..fb733ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ## Git ### Added +- SC2316: Warn about 'local readonly foo' and similar (thanks, patrickxia!) ### Fixed From 9aa4c22aa6fb54d511148bdc8b135e353529ffcc Mon Sep 17 00:00:00 2001 From: Frazer Smith Date: Mon, 16 May 2022 06:56:46 +0000 Subject: [PATCH 535/763] ci: update github actions --- .github/workflows/build.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5595219..a435cf4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,7 +15,7 @@ jobs: sudo apt-get install cabal-install - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 @@ -37,7 +37,7 @@ jobs: mv dist-newstyle/sdist/*.tar.gz source/source.tar.gz - name: Upload artifact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: source path: source/ @@ -51,10 +51,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Download artifacts - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 - name: Build source run: | @@ -63,7 +63,7 @@ jobs: ( cd bin && ../build/run_builder ../source/source.tar.gz ../build/${{matrix.build}} ) - name: Upload artifact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: bin path: bin/ @@ -74,10 +74,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Download artifacts - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 - name: Work around GitHub permissions bug run: chmod +x bin/*/shellcheck* @@ -92,7 +92,7 @@ jobs: rm -rf */ README* LICENSE* - name: Upload artifact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: deploy path: deploy/ @@ -104,10 +104,10 @@ jobs: environment: Deploy steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Download artifacts - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 - name: Upload to GitHub env: From 7ceb1f15193f43109a547ccc3d8ba74faaa72ade Mon Sep 17 00:00:00 2001 From: ygeyzel Date: Sun, 17 Jul 2022 21:46:42 +0300 Subject: [PATCH 536/763] SC2183 grammer fix: 'variable' instead of 'variables' if only one variable --- src/ShellCheck/Checks/Commands.hs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index e65dc68..d8635ea 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -675,6 +675,7 @@ checkPrintfVar = CommandCheck (Exactly "printf") (f . arguments) where let formats = getPrintfFormats string let formatCount = length formats let argCount = length more + let pluraliseIfMany word n = if n > 1 then word ++ "s" else word return $ if | argCount == 0 && formatCount == 0 -> @@ -690,7 +691,8 @@ checkPrintfVar = CommandCheck (Exactly "printf") (f . arguments) where return () -- Great: a suitable number of arguments | otherwise -> warn (getId format) 2183 $ - "This format string has " ++ show formatCount ++ " variables, but is passed " ++ show argCount ++ " arguments." + "This format string has " ++ show formatCount ++ " " ++ (pluraliseIfMany "variable" formatCount) ++ + ", but is passed " ++ show argCount ++ " arguments." unless ('%' `elem` concat (oversimplify format) || isLiteral format) $ info (getId format) 2059 From 363c0633e0525e6e7d1714cac83e420875345b85 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Fri, 11 Feb 2022 17:17:04 -0800 Subject: [PATCH 537/763] When reparsing array indices, do it recursively --- src/ShellCheck/Analytics.hs | 1 + src/ShellCheck/Parser.hs | 9 +++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index e6e47ea..7da5786 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2394,6 +2394,7 @@ prop_checkUnused47= verifyNotTree checkUnusedAssignments "a=1; alias hello='echo prop_checkUnused48= verifyNotTree checkUnusedAssignments "_a=1" prop_checkUnused49= verifyNotTree checkUnusedAssignments "declare -A array; key=a; [[ -v array[$key] ]]" prop_checkUnused50= verifyNotTree checkUnusedAssignments "foofunc() { :; }; typeset -fx foofunc" +prop_checkUnused51= verifyTree checkUnusedAssignments "x[y[z=1]]=1; echo ${x[@]}" checkUnusedAssignments params t = execWriter (mapM_ warnFor unused) where diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 8fbdb5a..833918c 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -3463,9 +3463,9 @@ notesForContext list = zipWith ($) [first, second] $ filter isName list -- Go over all T_UnparsedIndex and reparse them as either arithmetic or text -- depending on declare -A statements. -reparseIndices root = - analyze blank blank f root +reparseIndices root = process root where + process = analyze blank blank f associative = getAssociativeArrays root isAssociative s = s `elem` associative f (T_Assignment id mode name indices value) = do @@ -3490,8 +3490,9 @@ reparseIndices root = fixAssignmentIndex name word = case word of - T_UnparsedIndex id pos src -> - parsed name pos src + T_UnparsedIndex id pos src -> do + idx <- parsed name pos src + process idx -- Recursively parse for cases like x[y[z=1]]=1 _ -> return word parsed name pos src = From a4042f752399b0aff032692331e9f561c2cb836c Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 18 Jul 2022 22:12:31 -0700 Subject: [PATCH 538/763] Parse &&/|| as left-associative --- src/ShellCheck/Analytics.hs | 2 +- src/ShellCheck/Parser.hs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 7da5786..f5ff4df 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -867,7 +867,7 @@ prop_checkShorthandIf5 = verifyNot checkShorthandIf "foo && rm || printf b" prop_checkShorthandIf6 = verifyNot checkShorthandIf "if foo && bar || baz; then true; fi" prop_checkShorthandIf7 = verifyNot checkShorthandIf "while foo && bar || baz; do true; done" prop_checkShorthandIf8 = verify checkShorthandIf "if true; then foo && bar || baz; fi" -checkShorthandIf params x@(T_AndIf id _ (T_OrIf _ _ (T_Pipeline _ _ t))) +checkShorthandIf params x@(T_OrIf _ (T_AndIf id _ _) (T_Pipeline _ _ t)) | not (isOk t || inCondition) = info id 2015 "Note that A && B || C is not if-then-else. C may run when A is true." where diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 833918c..3958406 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -2286,7 +2286,7 @@ readAndOr = do parseProblemAt apos ErrorC 1123 "ShellCheck directives are only valid in front of complete compound commands, like 'if', not e.g. individual 'elif' branches." andOr <- withAnnotations annotations $ - chainr1 readPipeline $ do + chainl1 readPipeline $ do op <- g_AND_IF <|> g_OR_IF readLineBreak return $ case op of T_AND_IF id -> T_AndIf id From c3bce51de38fa57b006b932661bdb6237e0c5db5 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 19 Jul 2022 17:45:54 -0700 Subject: [PATCH 539/763] Allow text to build on Fedora by installing dependencies --- test/distrotest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/distrotest b/test/distrotest index 50a5a17..464768c 100755 --- a/test/distrotest +++ b/test/distrotest @@ -63,7 +63,7 @@ debian:testing apt-get update && apt-get install -y cabal-install ubuntu:latest apt-get update && apt-get install -y cabal-install haskell:latest true opensuse/leap:latest zypper install -y cabal-install ghc -fedora:latest dnf install -y cabal-install ghc-template-haskell-devel findutils +fedora:latest dnf install -y cabal-install ghc-template-haskell-devel findutils libstdc++-static gcc-c++ archlinux:latest pacman -S -y --noconfirm cabal-install ghc-static base-devel # Ubuntu LTS From cc04b4011967f873e14c0e6ad506ca37bed1f97f Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 19 Jul 2022 18:22:11 -0700 Subject: [PATCH 540/763] Freeze macOS dependency by sha256 --- build/darwin.x86_64/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/build/darwin.x86_64/Dockerfile b/build/darwin.x86_64/Dockerfile index ecd1cad..9e33a82 100644 --- a/build/darwin.x86_64/Dockerfile +++ b/build/darwin.x86_64/Dockerfile @@ -1,5 +1,4 @@ -# DIGEST:sha256:fa32af4677e2860a1c5950bc8c360f309e2a87e2ddfed27b642fddf7a6093b76 -FROM liushuyu/osxcross:latest +FROM liushuyu/osxcross@sha256:fa32af4677e2860a1c5950bc8c360f309e2a87e2ddfed27b642fddf7a6093b76 ENV TARGET x86_64-apple-darwin18 ENV TARGETNAME darwin.x86_64 From 7946bf5657905bba74be360bf16b287008ed1bdb Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 19 Jul 2022 21:40:03 -0700 Subject: [PATCH 541/763] Upgrade cURL for Windows build image --- build/windows.x86_64/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/windows.x86_64/Dockerfile b/build/windows.x86_64/Dockerfile index 11e67e8..1e5c5d9 100644 --- a/build/windows.x86_64/Dockerfile +++ b/build/windows.x86_64/Dockerfile @@ -12,7 +12,7 @@ WORKDIR /haskell RUN curl -L "https://downloads.haskell.org/~ghc/8.10.4/ghc-8.10.4-x86_64-unknown-mingw32.tar.xz" | tar xJ --strip-components=1 WORKDIR /haskell/bin RUN curl -L "https://downloads.haskell.org/~cabal/cabal-install-3.2.0.0/cabal-install-3.2.0.0-x86_64-unknown-mingw32.zip" | busybox unzip - -RUN curl -L "https://curl.se/windows/dl-7.75.0/curl-7.75.0-win64-mingw.zip" | busybox unzip - && mv curl-7.75.0-win64-mingw/bin/* . +RUN curl -L "https://curl.se/windows/dl-7.84.0/curl-7.84.0-win64-mingw.zip" | busybox unzip - && mv curl-7.84.0-win64-mingw/bin/* . ENV WINEPATH /haskell/bin # It's unknown whether Cabal on Windows suffers from the same issue From f77a545282f02010a0c52b66bd6ee5e860c2b314 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 19 Jul 2022 14:23:27 -0700 Subject: [PATCH 542/763] Control Flow Graph / Data Flow Analysis support --- CHANGELOG.md | 4 + ShellCheck.cabal | 8 + src/ShellCheck/ASTLib.hs | 116 ++- src/ShellCheck/Analytics.hs | 2 + src/ShellCheck/Analyzer.hs | 5 +- src/ShellCheck/AnalyzerLib.hs | 142 +-- src/ShellCheck/CFG.hs | 1147 +++++++++++++++++++++++++ src/ShellCheck/CFGAnalysis.hs | 1113 ++++++++++++++++++++++++ src/ShellCheck/Checks/Commands.hs | 1 + src/ShellCheck/Checks/ControlFlow.hs | 101 +++ src/ShellCheck/Checks/ShellSupport.hs | 1 + src/ShellCheck/Data.hs | 29 +- src/ShellCheck/Debug.hs | 313 +++++++ src/ShellCheck/Fixer.hs | 3 +- src/ShellCheck/Parser.hs | 5 +- src/ShellCheck/Prelude.hs | 48 ++ test/shellcheck.hs | 6 + 17 files changed, 2909 insertions(+), 135 deletions(-) create mode 100644 src/ShellCheck/CFG.hs create mode 100644 src/ShellCheck/CFGAnalysis.hs create mode 100644 src/ShellCheck/Checks/ControlFlow.hs create mode 100644 src/ShellCheck/Debug.hs create mode 100644 src/ShellCheck/Prelude.hs diff --git a/CHANGELOG.md b/CHANGELOG.md index fb733ce..4763ddd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ ### Fixed ### Changed +- ShellCheck now has a Data Flow Analysis engine to make smarter decisions + based on control flow rather than just syntax. Existing checks will + gradually start using it, which may cause them to trigger differently + (but more accurately). ## v0.8.0 - 2021-11-06 diff --git a/ShellCheck.cabal b/ShellCheck.cabal index 1167c82..b22b5c8 100644 --- a/ShellCheck.cabal +++ b/ShellCheck.cabal @@ -53,6 +53,7 @@ library deepseq >= 1.4.0.0, Diff >= 0.2.0, directory >= 1.2.3.0, + fgl, mtl >= 2.2.1, filepath, parsec, @@ -66,11 +67,15 @@ library ShellCheck.Analytics ShellCheck.Analyzer ShellCheck.AnalyzerLib + ShellCheck.CFG + ShellCheck.CFGAnalysis ShellCheck.Checker ShellCheck.Checks.Commands + ShellCheck.Checks.ControlFlow ShellCheck.Checks.Custom ShellCheck.Checks.ShellSupport ShellCheck.Data + ShellCheck.Debug ShellCheck.Fixer ShellCheck.Formatter.Format ShellCheck.Formatter.CheckStyle @@ -82,6 +87,7 @@ library ShellCheck.Formatter.Quiet ShellCheck.Interface ShellCheck.Parser + ShellCheck.Prelude ShellCheck.Regex other-modules: Paths_ShellCheck @@ -100,6 +106,7 @@ executable shellcheck deepseq >= 1.4.0.0, Diff >= 0.2.0, directory >= 1.2.3.0, + fgl, mtl >= 2.2.1, filepath, parsec >= 3.0, @@ -120,6 +127,7 @@ test-suite test-shellcheck deepseq >= 1.4.0.0, Diff >= 0.2.0, directory >= 1.2.3.0, + fgl, mtl >= 2.2.1, filepath, parsec, diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 7c88432..7cc5af2 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -21,6 +21,7 @@ module ShellCheck.ASTLib where import ShellCheck.AST +import ShellCheck.Prelude import ShellCheck.Regex import Control.Monad.Writer @@ -138,7 +139,7 @@ getFlagsUntil stopCondition (T_SimpleCommand _ _ (_:args)) = flag (x, '-':'-':arg) = [ (x, takeWhile (/= '=') arg) ] flag (x, '-':args) = map (\v -> (x, [v])) args flag (x, _) = [ (x, "") ] -getFlagsUntil _ _ = error "Internal shellcheck error, please report! (getFlags on non-command)" +getFlagsUntil _ _ = error $ pleaseReport "getFlags on non-command" -- Get all flags in a GNU way, up until -- getAllFlags :: Token -> [(Token, String)] @@ -785,5 +786,118 @@ executableFromShebang = shellFor basename s = reverse . takeWhile (/= '/') . reverse $ s skipFlags = dropWhile ("-" `isPrefixOf`) + +-- Determining if a name is a variable +isVariableStartChar x = x == '_' || isAsciiLower x || isAsciiUpper x +isVariableChar x = isVariableStartChar x || isDigit x +isSpecialVariableChar = (`elem` "*@#?-$!") +variableNameRegex = mkRegex "[_a-zA-Z][_a-zA-Z0-9]*" + +prop_isVariableName1 = isVariableName "_fo123" +prop_isVariableName2 = not $ isVariableName "4" +prop_isVariableName3 = not $ isVariableName "test: " +isVariableName (x:r) = isVariableStartChar x && all isVariableChar r +isVariableName _ = False + + +-- Get the variable name from an expansion like ${var:-foo} +prop_getBracedReference1 = getBracedReference "foo" == "foo" +prop_getBracedReference2 = getBracedReference "#foo" == "foo" +prop_getBracedReference3 = getBracedReference "#" == "#" +prop_getBracedReference4 = getBracedReference "##" == "#" +prop_getBracedReference5 = getBracedReference "#!" == "!" +prop_getBracedReference6 = getBracedReference "!#" == "#" +prop_getBracedReference7 = getBracedReference "!foo#?" == "foo" +prop_getBracedReference8 = getBracedReference "foo-bar" == "foo" +prop_getBracedReference9 = getBracedReference "foo:-bar" == "foo" +prop_getBracedReference10= getBracedReference "foo: -1" == "foo" +prop_getBracedReference11= getBracedReference "!os*" == "" +prop_getBracedReference11b= getBracedReference "!os@" == "" +prop_getBracedReference12= getBracedReference "!os?bar**" == "" +prop_getBracedReference13= getBracedReference "foo[bar]" == "foo" +getBracedReference s = fromMaybe s $ + nameExpansion s `mplus` takeName noPrefix `mplus` getSpecial noPrefix `mplus` getSpecial s + where + noPrefix = dropPrefix s + dropPrefix (c:rest) | c `elem` "!#" = rest + dropPrefix cs = cs + takeName s = do + let name = takeWhile isVariableChar s + guard . not $ null name + return name + getSpecial (c:_) | isSpecialVariableChar c = return [c] + getSpecial _ = fail "empty or not special" + + nameExpansion ('!':next:rest) = do -- e.g. ${!foo*bar*} + guard $ isVariableChar next -- e.g. ${!@} + first <- find (not . isVariableChar) rest + guard $ first `elem` "*?@" + return "" + nameExpansion _ = Nothing + +-- Get the variable modifier like /a/b in ${var/a/b} +prop_getBracedModifier1 = getBracedModifier "foo:bar:baz" == ":bar:baz" +prop_getBracedModifier2 = getBracedModifier "!var:-foo" == ":-foo" +prop_getBracedModifier3 = getBracedModifier "foo[bar]" == "[bar]" +prop_getBracedModifier4 = getBracedModifier "foo[@]@Q" == "[@]@Q" +prop_getBracedModifier5 = getBracedModifier "@@Q" == "@Q" +getBracedModifier s = headOrDefault "" $ do + let var = getBracedReference s + a <- dropModifier s + dropPrefix var a + where + dropPrefix [] t = return t + dropPrefix (a:b) (c:d) | a == c = dropPrefix b d + dropPrefix _ _ = [] + + dropModifier (c:rest) | c `elem` "#!" = [rest, c:rest] + dropModifier x = [x] + +-- Get the variables from indices like ["x", "y"] in ${var[x+y+1]} +prop_getIndexReferences1 = getIndexReferences "var[x+y+1]" == ["x", "y"] +getIndexReferences s = fromMaybe [] $ do + match <- matchRegex re s + index <- match !!! 0 + return $ matchAllStrings variableNameRegex index + where + re = mkRegex "(\\[.*\\])" + +prop_getOffsetReferences1 = getOffsetReferences ":bar" == ["bar"] +prop_getOffsetReferences2 = getOffsetReferences ":bar:baz" == ["bar", "baz"] +prop_getOffsetReferences3 = getOffsetReferences "[foo]:bar" == ["bar"] +prop_getOffsetReferences4 = getOffsetReferences "[foo]:bar:baz" == ["bar", "baz"] +getOffsetReferences mods = fromMaybe [] $ do +-- if mods start with [, then drop until ] + match <- matchRegex re mods + offsets <- match !!! 1 + return $ matchAllStrings variableNameRegex offsets + where + re = mkRegex "^(\\[.+\\])? *:([^-=?+].*)" + + +-- Returns whether a token is a parameter expansion without any modifiers. +-- True for $var ${var} $1 $# +-- False for ${#var} ${var[x]} ${var:-0} +isUnmodifiedParameterExpansion t = + case t of + T_DollarBraced _ False _ -> True + T_DollarBraced _ _ list -> + let str = concat $ oversimplify list + in getBracedReference str == str + _ -> False + +--- A list of the element and all its parents up to the root node. +getPath tree t = t : + case Map.lookup (getId t) tree of + Nothing -> [] + Just parent -> getPath tree parent + +isClosingFileOp op = + case op of + T_IoDuplicate _ (T_GREATAND _) "-" -> True + T_IoDuplicate _ (T_LESSAND _) "-" -> True + _ -> False + + return [] runTests = $quickCheckAll diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index f5ff4df..bf5d179 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -24,8 +24,10 @@ module ShellCheck.Analytics (runAnalytics, optionalChecks, ShellCheck.Analytics. import ShellCheck.AST import ShellCheck.ASTLib import ShellCheck.AnalyzerLib hiding (producesComments) +import qualified ShellCheck.CFGAnalysis as CF import ShellCheck.Data import ShellCheck.Parser +import ShellCheck.Prelude import ShellCheck.Interface import ShellCheck.Regex diff --git a/src/ShellCheck/Analyzer.hs b/src/ShellCheck/Analyzer.hs index eb231c2..ff2e457 100644 --- a/src/ShellCheck/Analyzer.hs +++ b/src/ShellCheck/Analyzer.hs @@ -25,6 +25,7 @@ import ShellCheck.Interface import Data.List import Data.Monoid import qualified ShellCheck.Checks.Commands +import qualified ShellCheck.Checks.ControlFlow import qualified ShellCheck.Checks.Custom import qualified ShellCheck.Checks.ShellSupport @@ -42,11 +43,13 @@ analyzeScript spec = newAnalysisResult { checkers spec params = mconcat $ map ($ params) [ ShellCheck.Checks.Commands.checker spec, + ShellCheck.Checks.ControlFlow.checker spec, ShellCheck.Checks.Custom.checker, ShellCheck.Checks.ShellSupport.checker ] optionalChecks = mconcat $ [ ShellCheck.Analytics.optionalChecks, - ShellCheck.Checks.Commands.optionalChecks + ShellCheck.Checks.Commands.optionalChecks, + ShellCheck.Checks.ControlFlow.optionalChecks ] diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 67c35b4..e998f2c 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -23,9 +23,11 @@ module ShellCheck.AnalyzerLib where import ShellCheck.AST import ShellCheck.ASTLib +import qualified ShellCheck.CFGAnalysis as CF import ShellCheck.Data import ShellCheck.Interface import ShellCheck.Parser +import ShellCheck.Prelude import ShellCheck.Regex import Control.Arrow (first) @@ -96,7 +98,9 @@ data Parameters = Parameters { -- The root node of the AST rootNode :: Token, -- map from token id to start and end position - tokenPositions :: Map.Map Id (Position, Position) + tokenPositions :: Map.Map Id (Position, Position), + -- Result from Control Flow Graph analysis (including data flow analysis) + cfgAnalysis :: CF.CFGAnalysis } deriving (Show) -- TODO: Cache results of common AST ops here @@ -189,8 +193,9 @@ makeCommentWithFix severity id code str fix = } in force withFix -makeParameters spec = - let params = Parameters { +makeParameters spec = params + where + params = Parameters { rootNode = root, shellType = fromMaybe (determineShell (asFallbackShell spec) root) $ asShellType spec, hasSetE = containsSetE root, @@ -215,9 +220,14 @@ makeParameters spec = shellTypeSpecified = isJust (asShellType spec) || isJust (asFallbackShell spec), parentMap = getParentTree root, variableFlow = getVariableFlow params root, - tokenPositions = asTokenPositions spec - } in params - where root = asScript spec + tokenPositions = asTokenPositions spec, + cfgAnalysis = CF.analyzeControlFlow cfParams root + } + cfParams = CF.CFGParameters { + CF.cfLastpipe = hasLastpipe params, + CF.cfPipefail = hasPipefail params + } + root = asScript spec -- Does this script mention 'set -e' anywhere? @@ -408,12 +418,6 @@ usedAsCommandName tree token = go (getId token) (tail $ getPath tree token) getId word == currentId || getId (getCommandTokenOrThis t) == currentId go _ _ = False --- A list of the element and all its parents up to the root node. -getPath tree t = t : - case Map.lookup (getId t) tree of - Nothing -> [] - Just parent -> getPath tree parent - -- Version of the above taking the map from the current context -- Todo: give this the name "getPath" getPathM t = do @@ -559,12 +563,6 @@ getModifiedVariables t = return (place, t, str, DataString SourceChecked) _ -> Nothing -isClosingFileOp op = - case op of - T_IoDuplicate _ (T_GREATAND _) "-" -> True - T_IoDuplicate _ (T_LESSAND _) "-" -> True - _ -> False - -- Consider 'export/declare -x' a reference, since it makes the var available getReferencedVariableCommand base@(T_SimpleCommand _ _ (T_NormalWord _ (T_Literal _ x:_):rest)) = @@ -746,13 +744,6 @@ getModifiedVariableCommand base@(T_SimpleCommand id cmdPrefix (T_NormalWord _ (T getModifiedVariableCommand _ = [] -getIndexReferences s = fromMaybe [] $ do - match <- matchRegex re s - index <- match !!! 0 - return $ matchAllStrings variableNameRegex index - where - re = mkRegex "(\\[.*\\])" - -- Given a NormalWord like foo or foo[$bar], get foo. -- Primarily used to get references for [[ -v foo[bar] ]] getVariableForTestDashV :: Token -> Maybe String @@ -767,18 +758,6 @@ getVariableForTestDashV t = do -- in a non-constant expression (while filtering out foo$x[$y]) toStr _ = return "\0" -prop_getOffsetReferences1 = getOffsetReferences ":bar" == ["bar"] -prop_getOffsetReferences2 = getOffsetReferences ":bar:baz" == ["bar", "baz"] -prop_getOffsetReferences3 = getOffsetReferences "[foo]:bar" == ["bar"] -prop_getOffsetReferences4 = getOffsetReferences "[foo]:bar:baz" == ["bar", "baz"] -getOffsetReferences mods = fromMaybe [] $ do --- if mods start with [, then drop until ] - match <- matchRegex re mods - offsets <- match !!! 1 - return $ matchAllStrings variableNameRegex offsets - where - re = mkRegex "^(\\[.+\\])? *:([^-=?+].*)" - getReferencedVariables parents t = case t of T_DollarBraced id _ l -> let str = concat $ oversimplify l in @@ -857,17 +836,6 @@ isConfusedGlobRegex ('*':_) = True isConfusedGlobRegex [x,'*'] | x `notElem` "\\." = True isConfusedGlobRegex _ = False -isVariableStartChar x = x == '_' || isAsciiLower x || isAsciiUpper x -isVariableChar x = isVariableStartChar x || isDigit x -isSpecialVariableChar = (`elem` "*@#?-$!") -variableNameRegex = mkRegex "[_a-zA-Z][_a-zA-Z0-9]*" - -prop_isVariableName1 = isVariableName "_fo123" -prop_isVariableName2 = not $ isVariableName "4" -prop_isVariableName3 = not $ isVariableName "test: " -isVariableName (x:r) = isVariableStartChar x && all isVariableChar r -isVariableName _ = False - getVariablesFromLiteralToken token = getVariablesFromLiteral (getLiteralStringDef " " token) @@ -880,73 +848,6 @@ getVariablesFromLiteral string = where variableRegex = mkRegex "\\$\\{?([A-Za-z0-9_]+)" --- Get the variable name from an expansion like ${var:-foo} -prop_getBracedReference1 = getBracedReference "foo" == "foo" -prop_getBracedReference2 = getBracedReference "#foo" == "foo" -prop_getBracedReference3 = getBracedReference "#" == "#" -prop_getBracedReference4 = getBracedReference "##" == "#" -prop_getBracedReference5 = getBracedReference "#!" == "!" -prop_getBracedReference6 = getBracedReference "!#" == "#" -prop_getBracedReference7 = getBracedReference "!foo#?" == "foo" -prop_getBracedReference8 = getBracedReference "foo-bar" == "foo" -prop_getBracedReference9 = getBracedReference "foo:-bar" == "foo" -prop_getBracedReference10= getBracedReference "foo: -1" == "foo" -prop_getBracedReference11= getBracedReference "!os*" == "" -prop_getBracedReference11b= getBracedReference "!os@" == "" -prop_getBracedReference12= getBracedReference "!os?bar**" == "" -prop_getBracedReference13= getBracedReference "foo[bar]" == "foo" -getBracedReference s = fromMaybe s $ - nameExpansion s `mplus` takeName noPrefix `mplus` getSpecial noPrefix `mplus` getSpecial s - where - noPrefix = dropPrefix s - dropPrefix (c:rest) | c `elem` "!#" = rest - dropPrefix cs = cs - takeName s = do - let name = takeWhile isVariableChar s - guard . not $ null name - return name - getSpecial (c:_) | isSpecialVariableChar c = return [c] - getSpecial _ = fail "empty or not special" - - nameExpansion ('!':next:rest) = do -- e.g. ${!foo*bar*} - guard $ isVariableChar next -- e.g. ${!@} - first <- find (not . isVariableChar) rest - guard $ first `elem` "*?@" - return "" - nameExpansion _ = Nothing - -prop_getBracedModifier1 = getBracedModifier "foo:bar:baz" == ":bar:baz" -prop_getBracedModifier2 = getBracedModifier "!var:-foo" == ":-foo" -prop_getBracedModifier3 = getBracedModifier "foo[bar]" == "[bar]" -prop_getBracedModifier4 = getBracedModifier "foo[@]@Q" == "[@]@Q" -prop_getBracedModifier5 = getBracedModifier "@@Q" == "@Q" -getBracedModifier s = headOrDefault "" $ do - let var = getBracedReference s - a <- dropModifier s - dropPrefix var a - where - dropPrefix [] t = return t - dropPrefix (a:b) (c:d) | a == c = dropPrefix b d - dropPrefix _ _ = [] - - dropModifier (c:rest) | c `elem` "#!" = [rest, c:rest] - dropModifier x = [x] - --- Useful generic functions. - --- Get element 0 or a default. Like `head` but safe. -headOrDefault _ (a:_) = a -headOrDefault def _ = def - --- Get the last element or a default. Like `last` but safe. -lastOrDefault def [] = def -lastOrDefault _ list = last list - ---- Get element n of a list, or Nothing. Like `!!` but safe. -(!!!) list i = - case drop i list of - [] -> Nothing - (r:_) -> Just r -- Run a command if the shell is in the given list whenShell l c = do @@ -999,17 +900,6 @@ isBashLike params = Dash -> False Sh -> False --- Returns whether a token is a parameter expansion without any modifiers. --- True for $var ${var} $1 $# --- False for ${#var} ${var[x]} ${var:-0} -isUnmodifiedParameterExpansion t = - case t of - T_DollarBraced _ False _ -> True - T_DollarBraced _ _ list -> - let str = concat $ oversimplify list - in getBracedReference str == str - _ -> False - isTrueAssignmentSource c = case c of DataString SourceChecked -> False diff --git a/src/ShellCheck/CFG.hs b/src/ShellCheck/CFG.hs new file mode 100644 index 0000000..101a0d7 --- /dev/null +++ b/src/ShellCheck/CFG.hs @@ -0,0 +1,1147 @@ +{- + Copyright 2022 Vidar Holen + + This file is part of ShellCheck. + https://www.shellcheck.net + + ShellCheck is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + ShellCheck is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +-} +{-# LANGUAGE TemplateHaskell #-} +{-# LANGUAGE DeriveAnyClass, DeriveGeneric #-} + +-- Constructs a Control Flow Graph from an AST +module ShellCheck.CFG ( + CFNode (..), + CFEdge (..), + CFEffect (..), + CFStringPart (..), + CFVariableProp (..), + CFGResult (..), + CFValue (..), + CFGraph, + CFGParameters (..), + IdTagged (..), + buildGraph + , ShellCheck.CFG.runTests -- STRIP + ) + where + +import GHC.Generics (Generic) +import ShellCheck.AST +import ShellCheck.ASTLib +import ShellCheck.Interface +import ShellCheck.Prelude +import ShellCheck.Regex +import Control.DeepSeq +import Control.Monad +import Control.Monad.Identity +import Data.List hiding (map) +import Data.Maybe +import qualified Data.Map as M +import qualified Data.Set as S +import Control.Monad.RWS.Lazy +import Data.Graph.Inductive.Graph +import Data.Graph.Inductive.Query.DFS +import Data.Graph.Inductive.PatriciaTree as G +import Debug.Trace -- STRIP + +import Test.QuickCheck.All (forAllProperties) +import Test.QuickCheck.Test (quickCheckWithResult, stdArgs, maxSuccess) + + +-- Our basic Graph type +type CFGraph = G.Gr CFNode CFEdge + +-- Node labels in a Control Flow Graph +data CFNode = + -- A no-op node for structural purposes + CFStructuralNode + -- A no-op for graph inspection purposes + | CFEntryPoint String + -- Drop current prefix assignments + | CFDropPrefixAssignments + -- A node with a certain effect on program state + | CFApplyEffects [IdTagged CFEffect] + -- The execution of a command or function by literal string if possible + | CFExecuteCommand (Maybe String) + -- Execute a subshell. These are represented by disjoint graphs just like + -- functions, but they don't require any form of name resolution + | CFExecuteSubshell String Node Node + -- Assignment of $? + | CFSetExitCode Id + -- The virtual 'exit' at the natural end of a subshell + | CFImpliedExit + -- An exit statement resolvable at CFG build time + | CFResolvedExit + -- An exit statement only resolvable at DFA time + | CFUnresolvedExit + -- An unreachable node, serving as the unconnected end point of a range + | CFUnreachable + -- Assignment of $! + | CFSetBackgroundPid Id + deriving (Eq, Ord, Show, Generic, NFData) + +-- Edge labels in a Control Flow Graph +data CFEdge = + CFEErrExit + -- Regular control flow edge + | CFEFlow + -- An edge that a human might think exists (e.g. from a backgrounded process to its parent) + | CFEFalseFlow + -- An edge followed on exit + | CFEExit + deriving (Eq, Ord, Show, Generic, NFData) + +-- Actions we track +data CFEffect = + CFModifyProps String [CFVariableProp] + | CFReadVariable String + | CFWriteVariable String CFValue + | CFWriteGlobal String CFValue + | CFWriteLocal String CFValue + | CFWritePrefix String CFValue + | CFDefineFunction String Id Node Node + | CFUndefine String + | CFUndefineVariable String + | CFUndefineFunction String + | CFUndefineNameref String + -- Usage implies that this is an array (e.g. it's expanded with index) + | CFHintArray String + -- Operation implies that the variable will be defined (e.g. [ -z "$var" ]) + | CFHintDefined String + deriving (Eq, Ord, Show, Generic, NFData) + +data IdTagged a = IdTagged Id a + deriving (Eq, Ord, Show, Generic, NFData) + +-- Where a variable's value comes from +data CFValue = + -- The special 'uninitialized' value + CFValueUninitialized + -- An arbitrary array value + | CFValueArray + -- An arbitrary string value + | CFValueString + -- An arbitrary integer + | CFValueInteger + -- Token 'Id' concatenates and assigns the given parts + | CFValueComputed Id [CFStringPart] + deriving (Eq, Ord, Show, Generic, NFData) + +-- Simplified computed strings +data CFStringPart = + -- A known literal string value, like 'foo' + CFStringLiteral String + -- The contents of a variable, like $foo + | CFStringVariable String + -- An value that is unknown but an integer + | CFStringInteger + -- An unknown string value, for things we can't handle + | CFStringUnknown + deriving (Eq, Ord, Show, Generic, NFData) + +-- The properties of a variable +data CFVariableProp = CFVPExport | CFVPArray + deriving (Eq, Ord, Show, Generic, NFData) + +-- Options when generating CFG +data CFGParameters = CFGParameters { + -- Whether the last element in a pipeline runs in the current shell + cfLastpipe :: Bool, + -- Whether all elements in a pipeline count towards the exit status + cfPipefail :: Bool +} + +data CFGResult = CFGResult { + -- The graph itself + cfGraph :: CFGraph, + -- Map from Id to start/end node + cfIdToNode :: M.Map Id (Node, Node) +} + deriving (Show) + +buildGraph :: CFGParameters -> Token -> CFGResult +buildGraph params root = + let + (nextNode, base) = execRWS (buildRoot root) (newCFContext params) 0 + (nodes, edges, mapping) = +-- renumberTopologically $ + removeUnnecessaryStructuralNodes + base + in + CFGResult { + cfGraph = mkGraph nodes edges, + cfIdToNode = M.fromList mapping + } + +remapGraph remap (nodes, edges, mapping) = + ( + map (remapNode remap) nodes, + map (remapEdge remap) edges, + map (\(id, (a,b)) -> (id, (remapHelper remap a, remapHelper remap b))) mapping + ) + +prop_testRenumbering = + let + s = CFStructuralNode + before = ( + [(1,s), (3,s), (4, s), (8,s)], + [(1,3,CFEFlow), (3,4, CFEFlow), (4,8,CFEFlow)], + [(Id 0, (3,4))] + ) + after = ( + [(0,s), (1,s), (2,s), (3,s)], + [(0,1,CFEFlow), (1,2, CFEFlow), (2,3,CFEFlow)], + [(Id 0, (1,2))] + ) + in after == renumberGraph before + +-- Renumber the graph for prettiness, so there are no gaps in node numbers +renumberGraph g@(nodes, edges, mapping) = + let renumbering = M.fromList (flip zip [0..] $ sort $ map fst nodes) + in remapGraph renumbering g + +prop_testRenumberTopologically = + let + s = CFStructuralNode + before = ( + [(4,s), (2,s), (3, s)], + [(4,2,CFEFlow), (2,3, CFEFlow)], + [(Id 0, (4,2))] + ) + after = ( + [(0,s), (1,s), (2,s)], + [(0,1,CFEFlow), (1,2, CFEFlow)], + [(Id 0, (0,1))] + ) + in after == renumberTopologically before + +-- Renumber the graph in topological order +renumberTopologically g@(nodes, edges, mapping) = + let renumbering = M.fromList (flip zip [0..] $ topsort (mkGraph nodes edges :: CFGraph)) + in remapGraph renumbering g + +prop_testRemoveStructural = + let + s = CFStructuralNode + before = ( + [(1,s), (2,s), (3, s), (4,s)], + [(1,2,CFEFlow), (2,3, CFEFlow), (3,4,CFEFlow)], + [(Id 0, (2,3))] + ) + after = ( + [(1,s), (2,s), (4,s)], + [(1,2,CFEFlow), (2,4,CFEFlow)], + [(Id 0, (2,2))] + ) + in after == removeUnnecessaryStructuralNodes before + +-- Collapse structural nodes that just form long chains like x->x->x. +-- This way we can generate them with abandon, without making DFA slower. +-- +-- Note in particular that we can't remove a structural node x in +-- foo -> x -> bar , because then the pre/post-condition for tokens +-- previously pointing to x would be wrong. +removeUnnecessaryStructuralNodes (nodes, edges, mapping) = + remapGraph recursiveRemapping + ( + filter (\(n, _) -> n `M.notMember` recursiveRemapping) nodes, + filter (`S.notMember` edgesToCollapse) edges, + mapping + ) + where + regularEdges = filter isRegularEdge edges + inDegree = counter $ map (\(from,to,_) -> from) regularEdges + outDegree = counter $ map (\(from,to,_) -> to) regularEdges + structuralNodes = S.fromList $ map fst $ filter isStructural nodes + candidateNodes = S.filter isLinear structuralNodes + edgesToCollapse = S.fromList $ filter filterEdges regularEdges + + remapping :: M.Map Node Node + remapping = foldl' (\m (new, old) -> M.insert old new m) M.empty $ map orderEdge $ S.toList edgesToCollapse + recursiveRemapping = M.fromList $ map (\c -> (c, recursiveLookup remapping c)) $ M.keys remapping + + filterEdges (a,b,_) = + a `S.member` candidateNodes && b `S.member` candidateNodes + + orderEdge (a,b,_) = if a < b then (a,b) else (b,a) + counter = foldl' (\map key -> M.insertWith (+) key 1 map) M.empty + isRegularEdge (_, _, CFEFlow) = True + isRegularEdge _ = False + + recursiveLookup :: M.Map Node Node -> Node -> Node + recursiveLookup map node = + case M.lookup node map of + Nothing -> node + Just x -> recursiveLookup map x + + isStructural (node, label) = + case label of + CFStructuralNode -> True + _ -> False + + isLinear node = + M.findWithDefault 0 node inDegree == 1 + && M.findWithDefault 0 node outDegree == 1 + + +remapNode :: M.Map Node Node -> LNode CFNode -> LNode CFNode +remapNode m (node, label) = + (remapHelper m node, newLabel) + where + newLabel = case label of + CFApplyEffects effects -> CFApplyEffects (map (remapEffect m) effects) + CFExecuteSubshell s a b -> CFExecuteSubshell s (remapHelper m a) (remapHelper m b) +-- CFSubShellStart reason node -> CFSubShellStart reason (remapHelper m node) + + _ -> label + +remapEffect map old@(IdTagged id effect) = + case effect of + CFDefineFunction name id start end -> IdTagged id $ CFDefineFunction name id (remapHelper map start) (remapHelper map end) + _ -> old + +remapEdge :: M.Map Node Node -> LEdge CFEdge -> LEdge CFEdge +remapEdge map (from, to, label) = (remapHelper map from, remapHelper map to, label) +remapHelper map n = M.findWithDefault n n map + +data Range = Range Node Node + deriving (Eq, Show) + +data CFContext = CFContext { + cfIsCondition :: Bool, + cfIsFunction :: Bool, + cfLoopStack :: [(Node, Node)], + cfExitTarget :: Maybe Node, + cfReturnTarget :: Maybe Node, + cfParameters :: CFGParameters +} +newCFContext params = CFContext { + cfIsCondition = False, + cfIsFunction = False, + cfLoopStack = [], + cfExitTarget = Nothing, + cfReturnTarget = Nothing, + cfParameters = params +} + +-- The monad we generate a graph in +type CFM a = RWS CFContext ([LNode CFNode], [LEdge CFEdge], [(Id, (Node, Node))]) Int a + +newNode :: CFNode -> CFM Node +newNode label = do + n <- get + put (n+1) + tell ([(n, label)], [], []) + return n + +newNodeRange :: CFNode -> CFM Range +-- newNodeRange label = nodeToRange <$> newNode label +newNodeRange label = nodeToRange <$> newNode label + +-- Build a disjoint piece of the graph and return a CFExecuteSubshell. The Id is used purely for debug naming. +subshell :: Id -> String -> CFM Range -> CFM Range +subshell id reason p = do + start <- newNode $ CFEntryPoint $ "Subshell " ++ show id ++ ": " ++ reason + end <- newNode CFStructuralNode + middle <- local (\c -> c { cfExitTarget = Just end, cfReturnTarget = Just end}) p + linkRanges [nodeToRange start, middle, nodeToRange end] + newNodeRange $ CFExecuteSubshell reason start end + + +withFunctionScope p = do + end <- newNode CFStructuralNode + body <- local (\c -> c { cfReturnTarget = Just end, cfIsFunction = True }) p + linkRanges [body, nodeToRange end] + + +nodeToRange :: Node -> Range +nodeToRange n = Range n n + +link :: Node -> Node -> CFEdge -> CFM () +link from to label = do + tell ([], [(from, to, label)], []) + +registerNode :: Id -> Range -> CFM () +registerNode id (Range start end) = tell ([], [], [(id, (start, end))]) + +linkRange :: Range -> Range -> CFM Range +linkRange = linkRangeAs CFEFlow + +linkRangeAs :: CFEdge -> Range -> Range -> CFM Range +linkRangeAs label (Range start mid1) (Range mid2 end) = do + link mid1 mid2 label + return (Range start end) + +-- Like linkRange but without actually linking +spanRange :: Range -> Range -> Range +spanRange (Range start mid1) (Range mid2 end) = Range start end + +linkRanges :: [Range] -> CFM Range +linkRanges [] = error "Empty range" +linkRanges (first:rest) = foldM linkRange first rest + +sequentially :: [Token] -> CFM Range +sequentially list = do + first <- newStructuralNode + rest <- mapM build list + linkRanges (first:rest) + +withContext :: (CFContext -> CFContext) -> CFM a -> CFM a +withContext = local + +withReturn :: Range -> CFM a -> CFM a +withReturn _ p = p + +asCondition :: CFM Range -> CFM Range +asCondition = withContext (\c -> c { cfIsCondition = True }) + +newStructuralNode = newNodeRange CFStructuralNode + +buildRoot :: Token -> CFM Range +buildRoot t = do + entry <- newNodeRange $ CFEntryPoint "MAIN" + impliedExit <- newNode CFImpliedExit + end <- newNode CFStructuralNode + start <- local (\c -> c { cfExitTarget = Just end, cfReturnTarget = Just impliedExit}) $ build t + range <- linkRanges [entry, start, nodeToRange impliedExit, nodeToRange end] + registerNode (getId t) range + return range + +applySingle e = CFApplyEffects [e] + +-- Build the CFG. +build :: Token -> CFM Range +build t = do + range <- build' t + registerNode (getId t) range + return range + where + build' t = case t of + T_Annotation _ _ list -> build list + T_Script _ _ list -> do + sequentially list + + TA_Assignment id op var@(TA_Variable _ name indices) rhs -> do + -- value first: (( var[x=1] = (x=2) )) runs x=1 last + value <- build rhs + subscript <- sequentially indices + read <- + if op == "=" + then none + -- This is += or something + else newNodeRange $ applySingle $ IdTagged id $ CFReadVariable name + + write <- newNodeRange $ applySingle $ IdTagged id $ CFWriteVariable name $ + if null indices + then CFValueInteger + else CFValueArray + + linkRanges [value, subscript, read, write] + + TA_Assignment id op lhs rhs -> do + -- This is likely an invalid assignment like (( 1 = 2 )), but it + -- could be e.g. x=y; (( $x = 3 )); echo $y, so expand both sides + -- without updating anything + sequentially [lhs, rhs] + + TA_Binary _ _ a b -> sequentially [a,b] + TA_Expansion _ list -> sequentially list + TA_Sequence _ list -> sequentially list + + TA_Trinary _ cond a b -> do + condition <- build cond + ifthen <- build a + elsethen <- build b + end <- newStructuralNode + linkRanges [condition, ifthen, end] + linkRanges [condition, elsethen, end] + + TA_Variable id name indices -> do + subscript <- sequentially indices + hint <- + if null indices + then none + else nodeToRange <$> newNode (applySingle $ IdTagged id $ CFHintArray name) + read <- nodeToRange <$> newNode (applySingle $ IdTagged id $ CFReadVariable name) + linkRanges [subscript, hint, read] + + TA_Unary id op (TA_Variable _ name indices) | "--" `isInfixOf` op || "++" `isInfixOf` op -> do + subscript <- sequentially indices + read <- newNodeRange $ applySingle $ IdTagged id $ CFReadVariable name + write <- newNodeRange $ applySingle $ IdTagged id $ CFWriteVariable name $ + if null indices + then CFValueInteger + else CFValueArray + linkRanges [subscript, read, write] + TA_Unary _ _ arg -> build arg + + TC_And _ SingleBracket _ lhs rhs -> do + sequentially [lhs, rhs] + + TC_And _ DoubleBracket _ lhs rhs -> do + left <- build lhs + right <- build rhs + end <- newStructuralNode + -- complete + linkRanges [left, right, end] + -- short circuit + linkRange left end + + -- TODO: Handle integer ops + TC_Binary _ mode str lhs rhs -> do + left <- build lhs + right <- build rhs + linkRange left right + + TC_Empty {} -> newStructuralNode + + TC_Group _ _ t -> build t + + -- TODO: Mark as checked + TC_Nullary _ _ arg -> build arg + + TC_Or _ SingleBracket _ lhs rhs -> sequentially [lhs, rhs] + + TC_Or _ DoubleBracket _ lhs rhs -> do + left <- build lhs + right <- build rhs + end <- newStructuralNode + -- complete + linkRanges [left, right, end] + -- short circuit + linkRange left end + + -- TODO: Handle -v, -z, -n + TC_Unary _ _ op arg -> do + build arg + + T_Arithmetic id root -> do + exe <- build root + status <- newNodeRange (CFSetExitCode id) + linkRange exe status + + T_AndIf _ lhs rhs -> do + left <- build lhs + right <- build rhs + end <- newStructuralNode + linkRange left right + linkRange right end + linkRange left end + + T_Array _ list -> sequentially list + + T_Assignment {} -> buildAssignment DefaultScope t + + T_Backgrounded id body -> do + start <- newStructuralNode + fork <- subshell id "backgrounding '&'" $ build body + pid <- newNodeRange $ CFSetBackgroundPid id + status <- newNodeRange $ CFSetExitCode id + + linkRange start fork + -- Add a join from the fork to warn about variable changes + linkRangeAs CFEFalseFlow fork pid + linkRanges [start, pid, status] + + T_Backticked id body -> + subshell id "`..` expansion" $ sequentially body + + T_Banged id cmd -> do + main <- build cmd + status <- newNodeRange (CFSetExitCode id) + linkRange main status + + T_BatsTest id _ body -> do + -- These are technically set by the 'run' command, but we'll just define them + -- up front to avoid figuring out which commands named "run" belong to Bats. + status <- newNodeRange $ applySingle $ IdTagged id $ CFWriteVariable "status" CFValueInteger + output <- newNodeRange $ applySingle $ IdTagged id $ CFWriteVariable "output" CFValueString + main <- build body + linkRanges [status, output, main] + + T_BraceExpansion _ list -> sequentially list + + T_BraceGroup id body -> + sequentially body + + T_CaseExpression id t [] -> build t + + T_CaseExpression id t list -> do + start <- newStructuralNode + token <- build t + branches <- mapM buildBranch list + end <- newStructuralNode + + let neighbors = zip branches $ tail branches + let (_, firstCond, _) = head branches + let (_, lastCond, lastBody) = last branches + + linkRange start token + linkRange token firstCond + mapM_ (uncurry $ linkBranch end) neighbors + linkRange lastBody end + + unless (any hasCatchAll list) $ + -- There's no *) branch, so assume we can fall through + void $ linkRange token end + + return $ spanRange start end + + where + -- for a | b | c, evaluate each in turn and allow short circuiting + buildCond list = do + start <- newStructuralNode + conds <- mapM build list + end <- newStructuralNode + linkRanges (start:conds) + mapM_ (`linkRange` end) conds + return $ spanRange start end + + buildBranch (typ, cond, body) = do + c <- buildCond cond + b <- sequentially body + linkRange c b + return (typ, c, b) + + linkBranch end (typ, cond, body) (_, nextCond, nextBody) = do + -- Failure case + linkRange cond nextCond + -- After body + case typ of + CaseBreak -> linkRange body end + CaseFallThrough -> linkRange body nextBody + CaseContinue -> linkRange body nextCond + + -- Find a *) if any + + hasCatchAll (_,cond,_) = any isCatchAll cond + isCatchAll c = fromMaybe False $ do + pg <- wordToExactPseudoGlob c + return $ pg `pseudoGlobIsSuperSetof` [PGMany] + + T_Condition _ _ op -> build op + + T_CoProc id maybeName t -> do + let name = fromMaybe "COPROC" maybeName + start <- newStructuralNode + parent <- newNodeRange $ applySingle $ IdTagged id $ CFWriteVariable name CFValueArray + child <- subshell id "coproc" $ build t + end <- newNodeRange $ CFSetExitCode id + + linkRange start parent + linkRange start child + linkRange parent end + linkRangeAs CFEFalseFlow child end + + return $ spanRange start end + T_CoProcBody _ t -> build t + + T_DollarArithmetic _ arith -> build arith + T_DollarDoubleQuoted _ list -> sequentially list + T_DollarSingleQuoted _ _ -> none + T_DollarBracket _ t -> build t + + T_DollarBraced id _ t -> do + let str = concat $ oversimplify t + let modifier = getBracedModifier str + let reference = getBracedReference str + let indices = getIndexReferences str + let offsets = getOffsetReferences str + vals <- build t + others <- mapM (\x -> nodeToRange <$> newNode (applySingle $ IdTagged id $ CFReadVariable x)) (indices ++ offsets) + deps <- linkRanges (vals:others) + read <- nodeToRange <$> newNode (applySingle $ IdTagged id $ CFReadVariable reference) + totalRead <- linkRange deps read + + if any (`isPrefixOf` modifier) ["=", ":="] + then do + optionalAssign <- newNodeRange (applySingle $ IdTagged id $ CFWriteVariable reference CFValueString) + result <- newStructuralNode + linkRange optionalAssign result + linkRange totalRead result + else return totalRead + + T_DoubleQuoted _ list -> sequentially list + + T_DollarExpansion id body -> + subshell id "$(..) expansion" $ sequentially body + + T_Extglob _ _ list -> sequentially list + + T_FdRedirect id ('{':identifier) op -> do + let name = takeWhile (/= '}') identifier + expression <- build op + rw <- newNodeRange $ + if isClosingFileOp op + then applySingle $ IdTagged id $ CFReadVariable name + else applySingle $ IdTagged id $ CFWriteVariable name CFValueInteger + + linkRange expression rw + + + T_FdRedirect _ name t -> do + build t + + T_ForArithmetic _ initT condT incT bodyT -> do + init <- build initT + cond <- build condT + body <- sequentially bodyT + inc <- build incT + end <- newStructuralNode + + -- Forward edges + linkRanges [init, cond, body, inc] + linkRange cond end + -- Backward edge + linkRange inc cond + return $ spanRange init end + + T_ForIn id name words body -> forInHelper id name words body + + -- For functions we generate an unlinked subgraph, and mention that in its definition node + T_Function id _ _ name body -> do + range <- local (\c -> c { cfExitTarget = Nothing }) $ do + entry <- newNodeRange $ CFEntryPoint $ "function " ++ name + f <- withFunctionScope $ build body + linkRange entry f + let (Range entry exit) = range + definition <- newNodeRange (applySingle $ IdTagged id $ CFDefineFunction name id entry exit) + exe <- newNodeRange (CFSetExitCode id) + linkRange definition exe + + T_Glob {} -> none + + T_HereString _ t -> build t + T_HereDoc _ _ _ _ list -> sequentially list + + T_IfExpression id ifs elses -> do + start <- newStructuralNode + branches <- doBranches start ifs elses [] + end <- newStructuralNode + mapM_ (`linkRange` end) branches + return $ spanRange start end + where + doBranches start ((conds, thens):rest) elses result = do + cond <- asCondition $ sequentially conds + action <- sequentially thens + linkRange start cond + linkRange cond action + doBranches cond rest elses (action:result) + doBranches start [] elses result = do + rest <- + if null elses + then newNodeRange (CFSetExitCode id) + else sequentially elses + linkRange start rest + return (rest:result) + + T_Include _ t -> build t + + T_IndexedElement _ indicesT valueT -> do + indices <- sequentially indicesT + value <- build valueT + linkRange indices value + + T_IoDuplicate _ op _ -> build op + + T_IoFile _ op t -> do + exp <- build t + doesntDoMuch <- build op + linkRange exp doesntDoMuch + + T_Literal {} -> none + + T_NormalWord _ list -> sequentially list + + T_OrIf _ lhs rhs -> do + left <- build lhs + right <- build rhs + end <- newStructuralNode + linkRange left right + linkRange right end + linkRange left end + + T_Pipeline _ _ [cmd] -> build cmd + T_Pipeline id _ cmds -> do + start <- newStructuralNode + hasLastpipe <- reader $ cfLastpipe . cfParameters + (leading, last) <- buildPipe hasLastpipe cmds + end <- newStructuralNode + + mapM_ (linkRange start) leading + mapM_ (\c -> linkRangeAs CFEFalseFlow c end) leading + linkRanges $ [start] ++ last ++ [end] + where + buildPipe True [x] = do + last <- build x + return ([], [last]) + buildPipe lp (first:rest) = do + this <- subshell id "pipeline" $ build first + (leading, last) <- buildPipe lp rest + return (this:leading, last) + buildPipe _ [] = return ([], []) + + T_ProcSub id op cmds -> do + start <- newStructuralNode + body <- subshell id (op ++ "() process substitution") $ sequentially cmds + end <- newStructuralNode + + linkRange start body + linkRangeAs CFEFalseFlow body end + linkRange start end + + T_Redirecting _ redirs cmd -> do + -- For simple commands, this is the other way around in bash + -- We do it in this order for comound commands like { x=name; } > "$x" + redir <- sequentially redirs + body <- build cmd + linkRange redir body + + T_SelectIn id name words body -> forInHelper id name words body + + T_SimpleCommand id vars [] -> do + -- Vars can also be empty, as in the command "> foo" + assignments <- sequentially vars + status <- newNodeRange (CFSetExitCode id) + linkRange assignments status + + T_SimpleCommand id vars list@(cmd:_) -> + handleCommand t vars list $ getUnquotedLiteral cmd + + T_SingleQuoted _ _ -> none + + T_SourceCommand _ originalCommand inlinedSource -> do + cmd <- build originalCommand + end <- newStructuralNode + inline <- withReturn end $ build inlinedSource + linkRange cmd inline + linkRange inline end + return $ spanRange cmd inline + + T_Subshell id body -> do + main <- subshell id "explicit (..) subshell" $ sequentially body + status <- newNodeRange (CFSetExitCode id) + linkRange main status + + T_UntilExpression id cond body -> whileHelper id cond body + T_WhileExpression id cond body -> whileHelper id cond body + + T_CLOBBER _ -> none + T_GREATAND _ -> none + T_LESSAND _ -> none + T_LESSGREAT _ -> none + T_DGREAT _ -> none + T_Greater _ -> none + T_Less _ -> none + T_ParamSubSpecialChar _ _ -> none + + x -> error ("Unimplemented: " ++ show x) + +-- Still in `where` clause + forInHelper id name words body = do + entry <- newStructuralNode + expansion <- sequentially words + assignmentChoice <- newStructuralNode + assignments <- + if null words || any willSplit words + then (:[]) <$> (newNodeRange $ applySingle $ IdTagged id $ CFWriteVariable name CFValueString) + else mapM (\t -> newNodeRange $ applySingle $ IdTagged id $ CFWriteVariable name $ CFValueComputed (getId t) $ tokenToParts t) words + body <- sequentially body + exit <- newStructuralNode + -- Forward edges + linkRanges [entry, expansion, assignmentChoice] + mapM_ (\t -> linkRanges [assignmentChoice, t, body]) assignments + linkRange body exit + linkRange expansion exit + -- Backward edge + linkRange body assignmentChoice + return $ spanRange entry exit + + whileHelper id cond body = do + condRange <- asCondition $ sequentially cond + bodyRange <- sequentially body + end <- newNodeRange (CFSetExitCode id) + + linkRange condRange bodyRange + linkRange bodyRange condRange + linkRange condRange end + + +handleCommand cmd vars args literalCmd = do + -- TODO: Handle assignments in declaring commands + + case literalCmd of + Just "exit" -> regularExpansion vars args $ handleExit + Just "return" -> regularExpansion vars args $ handleReturn + Just "unset" -> regularExpansionWithStatus vars args $ handleUnset args + + Just "declare" -> handleDeclare args + Just "local" -> handleDeclare args + Just "typeset" -> handleDeclare args + + Just "printf" -> regularExpansionWithStatus vars args $ handlePrintf args + Just "wait" -> regularExpansionWithStatus vars args $ handleWait args + + Just "mapfile" -> regularExpansionWithStatus vars args $ handleMapfile args + Just "readarray" -> regularExpansionWithStatus vars args $ handleMapfile args + + Just "DEFINE_boolean" -> regularExpansionWithStatus vars args $ handleDEFINE args + Just "DEFINE_float" -> regularExpansionWithStatus vars args $ handleDEFINE args + Just "DEFINE_integer" -> regularExpansionWithStatus vars args $ handleDEFINE args + Just "DEFINE_string" -> regularExpansionWithStatus vars args $ handleDEFINE args + + -- This will mostly behave like 'command' but ok + Just "builtin" -> + case args of + [_] -> regular + (_:newargs@(newcmd:_)) -> + handleCommand newcmd vars newargs $ getLiteralString newcmd + Just "command" -> + case args of + [_] -> regular + (_:newargs@(newcmd:_)) -> + handleOthers (getId newcmd) vars newargs $ getLiteralString newcmd + _ -> regular + + where + regular = handleOthers (getId cmd) vars args literalCmd + handleExit = do + exitNode <- reader cfExitTarget + case exitNode of + Just target -> do + exit <- newNode CFResolvedExit + link exit target CFEExit + unreachable <- newNode CFUnreachable + return $ Range exit unreachable + Nothing -> do + exit <- newNode CFUnresolvedExit + unreachable <- newNode CFUnreachable + return $ Range exit unreachable + + handleReturn = do + returnTarget <- reader cfReturnTarget + case returnTarget of + Nothing -> error $ pleaseReport "missing return target" + Just target -> do + ret <- newNode CFStructuralNode + link ret target CFEFlow + unreachable <- newNode CFUnreachable + return $ Range ret unreachable + + handleUnset (cmd:args) = do + case () of + _ | "n" `elem` flagNames -> unsetWith CFUndefineNameref + _ | "v" `elem` flagNames -> unsetWith CFUndefineVariable + _ | "f" `elem` flagNames -> unsetWith CFUndefineFunction + _ -> unsetWith CFUndefine + where + pairs :: [(String, Token)] -- [(Flag string, token)] e.g. [("-f", t), ("", myfunc)] + pairs = map (\(str, (flag, val)) -> (str, flag)) $ fromMaybe (map (\c -> ("", (c,c))) args) $ getGnuOpts "vfn" args + (names, flags) = partition (null . fst) pairs + flagNames = map fst flags + literalNames :: [(Token, String)] -- Literal names to unset, e.g. [(myfuncToken, "myfunc")] + literalNames = mapMaybe (\(_, t) -> getLiteralString t >>= (return . (,) t)) names + -- Apply a constructor like CFUndefineVariable to each literalName, and tag with its id + unsetWith c = newNodeRange $ CFApplyEffects $ map (\(token, name) -> IdTagged (getId token) $ c name) literalNames + + + variableAssignRegex = mkRegex "^([_a-zA-Z][_a-zA-Z0-9]*)=" + + handleDeclare (cmd:args) = do + isFunc <- asks cfIsFunction + let (evaluated, effects) = mconcat $ map (toEffects isFunc) args + before <- sequentially $ evaluated + effect <- newNodeRange $ CFApplyEffects effects + result <- newNodeRange $ CFSetExitCode (getId cmd) + linkRanges [before, effect, result] + where + opts = map fst $ getGenericOpts args + array = "a" `elem` opts || "A" `elem` opts + integer = "i" `elem` opts + func = "f" `elem` opts || "F" `elem` opts + global = "g" `elem` opts + writer isFunc = + case () of + _ | global -> CFWriteGlobal + _ | isFunc -> CFWriteLocal + _ -> CFWriteVariable + + toEffects :: Bool -> Token -> ([Token], [IdTagged CFEffect]) + toEffects isFunc (T_Assignment id mode var idx t) = + let + pre = idx ++ [t] + isArray = array || (not $ null idx) + asArray = [ IdTagged id $ (writer isFunc) var CFValueArray ] + asString = [ IdTagged id $ (writer isFunc) var $ + if integer + then CFValueInteger -- TODO: Also handle integer variable property + else CFValueComputed (getId t) $ [ CFStringVariable var | mode == Append ] ++ tokenToParts t + ] + in + (pre, if isArray then asArray else asString ) + + toEffects isFunc t = + let + pre = [t] + literal = fromJust $ getLiteralStringExt (const $ Just "\0") t + isKnown = '\0' `notElem` literal + match = fmap head $ variableAssignRegex `matchRegex` literal + name = fromMaybe literal match + + typer def = + if array + then CFValueArray + else + if integer + then CFValueInteger + else def + + asLiteral = [ + IdTagged (getId t) $ (writer isFunc) name $ + typer $ CFValueComputed (getId t) [ CFStringLiteral $ drop 1 $ dropWhile (/= '=') $ literal ] + ] + asUnknown = [ + IdTagged (getId t) $ (writer isFunc) name $ + typer $ CFValueString + ] + asBlank = [ + IdTagged (getId t) $ (writer isFunc) name $ + typer $ CFValueComputed (getId t) [] + ] + in + case () of + _ | not (isVariableName name) -> (pre, []) + _ | isJust match && isKnown -> (pre, asLiteral) + _ | isJust match -> (pre, asUnknown) + _ -> (pre, asBlank) + + handlePrintf (cmd:args) = + newNodeRange $ CFApplyEffects $ maybeToList findVar + where + findVar = do + flags <- getBsdOpts "v:" args + (flag, arg) <- lookup "v" flags + name <- getLiteralString arg + return $ IdTagged (getId arg) $ CFWriteVariable name CFValueString + + handleWait (cmd:args) = + newNodeRange $ CFApplyEffects $ maybeToList findVar + where + findVar = do + let flags = getGenericOpts args + (flag, arg) <- lookup "p" flags + name <- getLiteralString arg + return $ IdTagged (getId arg) $ CFWriteVariable name CFValueInteger + + handleMapfile (cmd:args) = + newNodeRange $ CFApplyEffects [findVar] + where + findVar = + let (id, name) = fromMaybe (getId cmd, "MAPFILE") $ getFromArg `mplus` getFromFallback + in IdTagged id $ CFWriteVariable name CFValueArray + + getFromArg = do + flags <- getGnuOpts "d:n:O:s:u:C:c:t" args + (_, arg) <- lookup "" flags + name <- getLiteralString arg + return (getId arg, name) + + getFromFallback = + listToMaybe $ mapMaybe getIfVar $ reverse args + getIfVar c = do + name <- getLiteralString c + guard $ isVariableName name + return (getId c, name) + + handleDEFINE (cmd:args) = + newNodeRange $ CFApplyEffects $ maybeToList findVar + where + findVar = do + name <- listToMaybe $ drop 1 args + str <- getLiteralString name + guard $ isVariableName str + return $ IdTagged (getId name) $ CFWriteVariable str CFValueString + + handleOthers id vars args cmd = + regularExpansion vars args $ do + exe <- newNodeRange $ CFExecuteCommand cmd + status <- newNodeRange $ CFSetExitCode id + linkRange exe status + + regularExpansion vars args p = do + args <- sequentially args + assignments <- mapM (buildAssignment PrefixScope) vars + exe <- p + dropAssignments <- + if null vars + then + return [] + else do + drop <- newNodeRange CFDropPrefixAssignments + return [drop] + + linkRanges $ [args] ++ assignments ++ [exe] ++ dropAssignments + + regularExpansionWithStatus vars args@(cmd:_) p = do + initial <- regularExpansion vars args p + status <- newNodeRange $ CFSetExitCode (getId cmd) + linkRange initial status + + +none = newStructuralNode + +data Scope = DefaultScope | GlobalScope | LocalScope | PrefixScope + +buildAssignment scope t = do + op <- case t of + T_Assignment id mode var indices value -> do + expand <- build value + index <- sequentially indices + read <- case mode of + Append -> newNodeRange (applySingle $ IdTagged id $ CFReadVariable var) + Assign -> none + let valueType = if null indices then f id value else CFValueArray + let scoper = + case scope of + PrefixScope -> CFWritePrefix + LocalScope -> CFWriteLocal + GlobalScope -> CFWriteGlobal + DefaultScope -> CFWriteVariable + write <- newNodeRange $ applySingle $ IdTagged id $ scoper var valueType + linkRanges [expand, index, read, write] + where + f :: Id -> Token -> CFValue + f id t@T_NormalWord {} = CFValueComputed id $ [CFStringVariable var | mode == Append] ++ tokenToParts t + f id t@(T_Literal _ str) = CFValueComputed id $ [CFStringVariable var | mode == Append] ++ tokenToParts t + f _ T_Array {} = CFValueArray + + registerNode (getId t) op + return op + + +tokenToParts t = + case t of + T_NormalWord _ list -> concatMap tokenToParts list + T_DoubleQuoted _ list -> concatMap tokenToParts list + T_SingleQuoted _ str -> [ CFStringLiteral str ] + T_Literal _ str -> [ CFStringLiteral str ] + T_DollarArithmetic {} -> [ CFStringInteger ] + T_DollarBracket {} -> [ CFStringInteger ] + T_DollarBraced _ _ list | isUnmodifiedParameterExpansion t -> [ CFStringVariable (getBracedReference $ concat $ oversimplify list) ] + -- Check if getLiteralString can handle it, if not it's unknown + _ -> [maybe CFStringUnknown CFStringLiteral $ getLiteralString t] + +return [] +runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) diff --git a/src/ShellCheck/CFGAnalysis.hs b/src/ShellCheck/CFGAnalysis.hs new file mode 100644 index 0000000..99ce450 --- /dev/null +++ b/src/ShellCheck/CFGAnalysis.hs @@ -0,0 +1,1113 @@ +{- + Copyright 2022 Vidar Holen + + This file is part of ShellCheck. + https://www.shellcheck.net + + ShellCheck is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + ShellCheck is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +-} +{-# LANGUAGE TemplateHaskell #-} +{-# LANGUAGE RankNTypes #-} +{-# LANGUAGE DeriveAnyClass, DeriveGeneric #-} + +{- + Data Flow Analysis on a Control Flow Graph. + + This module implements a pretty standard iterative Data Flow Analysis. + For an overview of the process, see Wikipedia. + + Since shell scripts rely heavily on global variables, this DFA includes + tracking the value of globals across calls. Each function invocation is + treated as a separate DFA problem, and a caching mechanism (hopefully) + avoids any exponential explosions. + + To do efficient DFA join operations (or merges, as the code calls them), + some of the data structures have an integer version attached. On update, + the version is changed. If two states have the same version number, + a merge is skipped on the grounds that they are identical. It is easy + to unintentionally forget to update/invalidate the version number, + and bugs will ensure. + + For performance reasons, the entire code runs in plain ST, with a manual + context object Ctx being passed around. It relies heavily on mutable + STRefs. However, this turned out to be literally thousands of times faster + than my several attempts using RWST, so it can't be helped. +-} + +module ShellCheck.CFGAnalysis ( + analyzeControlFlow + ,CFGParameters (..) + ,CFGAnalysis (..) + ,ProgramState (..) + ,VariableValue (..) + ,SpaceStatus (..) + ,getIncomingState + ,getOutgoingState + ,ShellCheck.CFGAnalysis.runTests -- STRIP + ) where + +import GHC.Generics (Generic) +import ShellCheck.AST +import ShellCheck.CFG +import qualified ShellCheck.Data as Data +import ShellCheck.Prelude +import Control.Monad +import Control.Monad.ST +import Control.DeepSeq +import Data.List hiding (map) +import Data.STRef +import Data.Maybe +import qualified Data.Map as M +import qualified Data.Set as S +import Data.Graph.Inductive.Graph +import Data.Graph.Inductive.Query.DFS +import Debug.Trace -- STRIP + +import Test.QuickCheck + + +iterationCount = 1000000 +cacheEntries = 10 + +-- The result of the data flow analysis +data CFGAnalysis = CFGAnalysis { + graph :: CFGraph, + tokenToNode :: M.Map Id (Node, Node), + nodeToData :: M.Map Node (ProgramState, ProgramState) +} deriving (Show, Generic, NFData) + +-- The program state we expose externally +data ProgramState = ProgramState { + variablesInScope :: M.Map String VariableValue, + stateIsReachable :: Bool +-- internalState :: InternalState +} deriving (Show, Eq, Generic, NFData) + +-- Conveniently get the state before a token id +getIncomingState :: CFGAnalysis -> Id -> Maybe ProgramState +getIncomingState analysis id = do + (start,end) <- M.lookup id $ tokenToNode analysis + fst <$> M.lookup start (nodeToData analysis) + +-- Conveniently get the state after a token id +getOutgoingState :: CFGAnalysis -> Id -> Maybe ProgramState +getOutgoingState analysis id = do + (start,end) <- M.lookup id $ tokenToNode analysis + snd <$> M.lookup end (nodeToData analysis) + +getDataForNode analysis node = M.lookup node $ nodeToData analysis + +-- The current state of data flow at a point in the program, potentially as a diff +data InternalState = InternalState { + sVersion :: Integer, + sGlobalValues :: VersionedMap String VariableValue, + sLocalValues :: VersionedMap String VariableValue, + sPrefixValues :: VersionedMap String VariableValue, + sFunctionTargets :: VersionedMap String FunctionValue, + sIsReachable :: Maybe Bool +} deriving (Show, Generic, NFData) + +newInternalState = InternalState { + sVersion = 0, + sGlobalValues = vmEmpty, + sLocalValues = vmEmpty, + sPrefixValues = vmEmpty, + sFunctionTargets = vmEmpty, + sIsReachable = Nothing +} + +unreachableState = modified newInternalState { + sIsReachable = Just False +} + +-- The default state we assume we get from the environment +createEnvironmentState :: InternalState +createEnvironmentState = do + foldl' (flip ($)) newInternalState $ concat [ + addVars Data.internalVariables unknownVariableValue, + addVars Data.variablesWithoutSpaces spacelessVariableValue, + addVars Data.specialIntegerVariables spacelessVariableValue + ] + where + addVars names val = map (\name -> insertGlobal name val) names + spacelessVariableValue = VariableValue { + literalValue = Nothing, + spaceStatus = SpaceStatusClean + } + + +modified s = s { sVersion = -1 } + +insertGlobal :: String -> VariableValue -> InternalState -> InternalState +insertGlobal name value state = modified state { + sGlobalValues = vmInsert name value $ sGlobalValues state +} + +insertLocal :: String -> VariableValue -> InternalState -> InternalState +insertLocal name value state = modified state { + sLocalValues = vmInsert name value $ sLocalValues state +} + +insertPrefix :: String -> VariableValue -> InternalState -> InternalState +insertPrefix name value state = modified state { + sPrefixValues = vmInsert name value $ sPrefixValues state +} + +insertFunction :: String -> FunctionValue -> InternalState -> InternalState +insertFunction name value state = modified state { + sFunctionTargets = vmInsert name value $ sFunctionTargets state +} + +internalToExternal :: InternalState -> ProgramState +internalToExternal s = + ProgramState { + -- Avoid introducing dependencies on the literal value as this is only for debugging purposes right now + variablesInScope = M.map (\c -> c { literalValue = Nothing }) flatVars, + -- internalState = s, -- For debugging + stateIsReachable = fromMaybe True $ sIsReachable s + } + where + flatVars = M.unionsWith (\_ last -> last) $ map mapStorage [sGlobalValues s, sLocalValues s, sPrefixValues s] + +-- Dependencies on values, e.g. "if there is a global variable named 'foo' without spaces" +-- This is used to see if the DFA of a function would result in the same state, so anything +-- that affects DFA must be tracked. +data StateDependency = + DepGlobalValue String VariableValue + | DepLocalValue String VariableValue + | DepPrefixValue String VariableValue + | DepFunction String (S.Set FunctionDefinition) + -- Whether invoking the node would result in recursion (i.e., is the function on the stack?) + | DepIsRecursive Node Bool + deriving (Show, Eq, Ord, Generic, NFData) + +-- A function definition, or lack thereof +data FunctionDefinition = FunctionUnknown | FunctionDefinition String Node Node + deriving (Show, Eq, Ord, Generic, NFData) + +-- The Set of places a command name can point (it's a Set to handle conditionally defined functions) +type FunctionValue = S.Set FunctionDefinition + +-- The scope of a function. ("Prefix" refers to e.g. `foo=1 env`) +data VariableScope = PrefixVar | LocalVar | GlobalVar + deriving (Show, Eq, Ord, Generic, NFData) + +-- Create an InternalState that fulfills the given dependencies +depsToState :: S.Set StateDependency -> InternalState +depsToState set = foldl insert newInternalState $ S.toList set + where + insert :: InternalState -> StateDependency -> InternalState + insert state dep = + case dep of + DepFunction name val -> insertFunction name val state + DepGlobalValue name val -> insertGlobal name val state + DepLocalValue name val -> insertLocal name val state + DepPrefixValue name val -> insertPrefix name val state + DepIsRecursive _ _ -> state + +unknownFunctionValue = S.singleton FunctionUnknown + +-- The information about the value of a single variable +data VariableValue = VariableValue { + literalValue :: Maybe String, -- TODO: For debugging. Remove me. + spaceStatus :: SpaceStatus +} + deriving (Show, Eq, Ord, Generic, NFData) + +-- Whether or not the value needs quoting (has spaces/globs), or we don't know +data SpaceStatus = SpaceStatusEmpty | SpaceStatusClean | SpaceStatusDirty deriving (Show, Eq, Ord, Generic, NFData) + + +unknownVariableValue = VariableValue { + literalValue = Nothing, + spaceStatus = SpaceStatusDirty +} + +emptyVariableValue = VariableValue { + literalValue = Just "", + spaceStatus = SpaceStatusEmpty +} + +mergeVariableValue a b = VariableValue { + literalValue = if literalValue a == literalValue b then literalValue a else Nothing, + spaceStatus = mergeSpaceStatus (spaceStatus a) (spaceStatus b) +} + +mergeSpaceStatus a b = + case (a,b) of + (SpaceStatusEmpty, y) -> y + (x, SpaceStatusEmpty) -> x + (SpaceStatusClean, SpaceStatusClean) -> SpaceStatusClean + _ -> SpaceStatusDirty + +-- A VersionedMap is a Map that keeps an additional integer version to quickly determine if it has changed. +-- * Version -1 means it's unknown (possibly and presumably changed) +-- * Version 0 means it's empty +-- * Version N means it's equal to any other map with Version N (this is required but not enforced) +data VersionedMap k v = VersionedMap { + mapVersion :: Integer, + mapStorage :: M.Map k v +} + deriving (Generic, NFData) + +-- This makes states more readable but inhibits copy-paste +instance (Show k, Show v) => Show (VersionedMap k v) where + show m = (if mapVersion m >= 0 then "V" ++ show (mapVersion m) else "U") ++ " " ++ show (mapStorage m) + +instance Eq InternalState where + (==) a b = stateIsQuickEqual a b || stateIsSlowEqual a b + +instance (Eq k, Eq v) => Eq (VersionedMap k v) where + (==) a b = vmIsQuickEqual a b || mapStorage a == mapStorage b + +instance (Ord k, Ord v) => Ord (VersionedMap k v) where + compare a b = + if vmIsQuickEqual a b + then EQ + else mapStorage a `compare` mapStorage b + + +-- A context with STRefs manually passed around to function. +-- This is done because it was dramatically much faster than any RWS type stack +data Ctx s = Ctx { + -- The current node + cNode :: STRef s Node, + -- The current input state + cInput :: STRef s InternalState, + -- The current output state + cOutput :: STRef s InternalState, + + -- The current functions/subshells stack + cStack :: [StackEntry s], + -- The input graph + cGraph :: CFGraph, + -- An incrementing counter to version maps + cCounter :: STRef s Integer, + -- A cache of input state dependencies to output effects + cCache :: STRef s (M.Map Node [(S.Set StateDependency, InternalState)]), + -- The states resulting from data flows per invocation path + cInvocations :: STRef s (M.Map [Node] (S.Set StateDependency, M.Map Node (InternalState, InternalState))) +} + +-- Whenever a function (or subshell) is invoked, a value like this is pushed onto the stack +data StackEntry s = StackEntry { + -- The entry point of this stack entry for the purpose of detecting recursion + entryPoint :: Node, + -- The node where this entry point was invoked + callSite :: Node, + -- A mutable set of dependencies we fetched from here or higher in the stack + dependencies :: STRef s (S.Set StateDependency), + -- The original input state for this stack entry + stackState :: InternalState +} + deriving (Eq, Generic, NFData) + + +-- Overwrite a base state with the contents of a diff state +-- This is unrelated to join/merge. +patchState :: InternalState -> InternalState -> InternalState +patchState base diff = + case () of + _ | sVersion diff == 0 -> base + _ | sVersion base == 0 -> diff + _ | stateIsQuickEqual base diff -> diff + _ -> + InternalState { + sVersion = -1, + sGlobalValues = vmPatch (sGlobalValues base) (sGlobalValues diff), + sLocalValues = vmPatch (sLocalValues base) (sLocalValues diff), + sPrefixValues = vmPatch (sPrefixValues base) (sPrefixValues diff), + sFunctionTargets = vmPatch (sFunctionTargets base) (sFunctionTargets diff), + sIsReachable = sIsReachable diff `mplus` sIsReachable base + } + +patchOutputM ctx diff = do + let cOut = cOutput ctx + oldState <- readSTRef cOut + let newState = patchState oldState diff + writeSTRef cOut newState + +-- Merge (aka Join) two states. This is monadic because it requires looking up +-- values from the current context. For example: +-- +-- f() { +-- foo || x=2 +-- HERE # This merge requires looking up the value of $x in the parent frame +-- } +-- x=1 +-- f +mergeState :: forall s. Ctx s -> InternalState -> InternalState -> ST s InternalState +mergeState ctx a b = do + -- Kludge: we want `readVariable` & friends not to read from an intermediate state, + -- so temporarily set a blank input. + let cin = cInput ctx + old <- readSTRef cin + writeSTRef cin newInternalState + x <- merge a b + writeSTRef cin old + return x + + where + + merge a b = + case () of + _ | sIsReachable a == Just True && sIsReachable b == Just False + || sIsReachable a == Just False && sIsReachable b == Just True -> + error $ pleaseReport "Unexpected merge of reachable and unreachable state" + _ | sIsReachable a == Just False && sIsReachable b == Just False -> + return unreachableState + _ | sVersion a >= 0 && sVersion b >= 0 && sVersion a == sVersion b -> return a + _ -> do + globals <- mergeMaps ctx mergeVariableValue readGlobal (sGlobalValues a) (sGlobalValues b) + locals <- mergeMaps ctx mergeVariableValue readVariable (sLocalValues a) (sLocalValues b) + prefix <- mergeMaps ctx mergeVariableValue readVariable (sPrefixValues a) (sPrefixValues b) + funcs <- mergeMaps ctx S.union readFunction (sFunctionTargets a) (sFunctionTargets b) + return $ InternalState { + sVersion = -1, + sGlobalValues = globals, + sLocalValues = locals, + sPrefixValues = prefix, + sFunctionTargets = funcs, + sIsReachable = liftM2 (&&) (sIsReachable a) (sIsReachable b) + } + +-- Merge a number of states, or return a default if there are no states +-- (it can't fold from newInternalState because this would be equivalent of adding a new input edge). +mergeStates :: forall s. Ctx s -> InternalState -> [InternalState] -> ST s InternalState +mergeStates ctx def list = + case list of + [] -> return def + (first:rest) -> foldM (mergeState ctx) first rest + +-- Merge two maps, key by key. If both maps have a key, the 'merger' is used. +-- If only one has the key, the 'reader' is used to fetch a second, and the two are merged as above. +mergeMaps :: (Ord k) => forall s. + Ctx s -> + (v -> v -> v) -> + (Ctx s -> k -> ST s v) -> + (VersionedMap k v) -> + (VersionedMap k v) -> + ST s (VersionedMap k v) +mergeMaps ctx merger reader a b = + if vmIsQuickEqual a b + then return a + else do + new <- M.fromDistinctAscList <$> reverse <$> f [] (M.toAscList $ mapStorage a) (M.toAscList $ mapStorage b) + vmFromMap ctx new + where + f l [] [] = return l + f l [] b = f l b [] + f l ((k,v):rest1) [] = do + other <- reader ctx k + f ((k, merger v other):l) rest1 [] + f l l1@((k1, v1):rest1) l2@((k2, v2):rest2) = + case k1 `compare` k2 of + EQ -> + f ((k1, merger v1 v2):l) rest1 rest2 + LT -> do + nv2 <- reader ctx k1 + f ((k1, merger v1 nv2):l) rest1 l2 + GT -> do + nv1 <- reader ctx k2 + f ((k2, merger nv1 v2):l) l1 rest2 + +vmFromMap ctx map = return $ VersionedMap { + mapVersion = -1, + mapStorage = map +} + +-- Give a VersionedMap a version if it does not already have one. +versionMap ctx map = + if mapVersion map >= 0 + then return map + else do + v <- nextVersion ctx + return map { + mapVersion = v + } + +-- Give an InternalState a version if it does not already have one. +versionState ctx state = + if sVersion state >= 0 + then return state + else do + self <- nextVersion ctx + ssGlobalValues <- versionMap ctx $ sGlobalValues state + ssLocalValues <- versionMap ctx $ sLocalValues state + ssFunctionTargets <- versionMap ctx $ sFunctionTargets state + return state { + sVersion = self, + sGlobalValues = ssGlobalValues, + sLocalValues = ssLocalValues, + sFunctionTargets = ssFunctionTargets + } + +-- Like 'not null' but for 2+ elements +is2plus :: [a] -> Bool +is2plus l = case l of + _:_:_ -> True + _ -> False + +-- Use versions to see if two states are trivially identical +stateIsQuickEqual a b = + let + va = sVersion a + vb = sVersion b + in + va >= 0 && vb >= 0 && va == vb + +-- A manual slow path 'Eq' (it's not derived because it's part of the custom Eq instance) +stateIsSlowEqual a b = + check sGlobalValues + && check sLocalValues + && check sPrefixValues + && check sFunctionTargets + && check sIsReachable + where + check f = f a == f b + +-- Check if two VersionedMaps are trivially equal +vmIsQuickEqual :: VersionedMap k v -> VersionedMap k v -> Bool +vmIsQuickEqual a b = + let + va = mapVersion a + vb = mapVersion b + in + va >= 0 && vb >= 0 && va == vb + +-- A new, empty VersionedMap +vmEmpty = VersionedMap { + mapVersion = 0, + mapStorage = M.empty +} + +-- Map.null for VersionedMaps +vmNull :: VersionedMap k v -> Bool +vmNull m = mapVersion m == 0 || (M.null $ mapStorage m) + +-- Map.lookup for VersionedMaps +vmLookup name map = M.lookup name $ mapStorage map + +-- Map.insert for VersionedMaps +vmInsert key val map = VersionedMap { + mapVersion = -1, + mapStorage = M.insert key val $ mapStorage map +} + +-- Overwrite all keys in the first map with values from the second +vmPatch :: (Ord k) => VersionedMap k v -> VersionedMap k v -> VersionedMap k v +vmPatch base diff = + case () of + _ | mapVersion base == 0 -> diff + _ | mapVersion diff == 0 -> base + _ | vmIsQuickEqual base diff -> diff + _ -> VersionedMap { + mapVersion = -1, + mapStorage = M.unionWith (flip const) (mapStorage base) (mapStorage diff) + } + +-- Modify a variable as with x=1. This applies it to the appropriate scope. +writeVariable :: forall s. Ctx s -> String -> VariableValue -> ST s () +writeVariable ctx name val = do + (_, typ) <- readVariableWithScope ctx name + case typ of + GlobalVar -> writeGlobal ctx name val + LocalVar -> writeLocal ctx name val + -- Prefixed variables actually become local variables in the invoked function + PrefixVar -> writeLocal ctx name val + +writeGlobal ctx name val = do + modifySTRef (cOutput ctx) $ insertGlobal name val + +writeLocal ctx name val = do + modifySTRef (cOutput ctx) $ insertLocal name val + +writePrefix ctx name val = do + modifySTRef (cOutput ctx) $ insertPrefix name val + +-- Look up a variable value, and also return its scope +readVariableWithScope :: forall s. Ctx s -> String -> ST s (VariableValue, VariableScope) +readVariableWithScope ctx name = lookupStack get dep def ctx name + where + def = (unknownVariableValue, GlobalVar) + get = getVariableWithScope + dep k v = + case v of + (val, GlobalVar) -> DepGlobalValue k val + (val, LocalVar) -> DepLocalValue k val + (val, PrefixVar) -> DepPrefixValue k val + +getVariableWithScope :: InternalState -> String -> Maybe (VariableValue, VariableScope) +getVariableWithScope s name = + case (vmLookup name $ sPrefixValues s, vmLookup name $ sLocalValues s, vmLookup name $ sGlobalValues s) of + (Just var, _, _) -> return (var, PrefixVar) + (_, Just var, _) -> return (var, LocalVar) + (_, _, Just var) -> return (var, GlobalVar) + _ -> Nothing + +undefineFunction ctx name = + writeFunction ctx name $ FunctionUnknown + +undefineVariable ctx name = + writeVariable ctx name $ emptyVariableValue + +readVariable ctx name = fst <$> readVariableWithScope ctx name + +readGlobal ctx name = lookupStack get dep def ctx name + where + def = unknownVariableValue + get s name = vmLookup name $ sGlobalValues s + dep k v = DepGlobalValue k v + +readFunction ctx name = lookupStack get dep def ctx name + where + def = unknownFunctionValue + get s name = vmLookup name $ sFunctionTargets s + dep k v = DepFunction k v + +writeFunction ctx name val = do + modifySTRef (cOutput ctx) $ insertFunction name $ S.singleton val + +-- Look up each state on the stack until a value is found (or the default is used), +-- then add this value as a StateDependency. +lookupStack :: forall s k v. + -- A function that maybe finds a value from a state + (InternalState -> k -> Maybe v) + -- A function that creates a dependency on what was found + -> (k -> v -> StateDependency) + -- A default value, if the value can't be found anywhere + -> v + -- Context + -> Ctx s + -- The key to look up + -> k + -- Returning the result + -> ST s v +lookupStack get dep def ctx key = do + top <- readSTRef $ cInput ctx + case get top key of + Just v -> return v + Nothing -> f (cStack ctx) + where + f [] = return def + f (s:rest) = do + -- Go up the stack until we find the value, and add + -- a dependency on each state (including where it was found) + res <- fromMaybe (f rest) (return <$> get (stackState s) key) + modifySTRef (dependencies s) $ S.insert $ dep key res + return res + +-- Like lookupStack but without adding dependencies +peekStack get def ctx key = do + top <- readSTRef $ cInput ctx + case get top key of + Just v -> return v + Nothing -> f (cStack ctx) + where + f [] = return def + f (s:rest) = + case get (stackState s) key of + Just v -> return v + Nothing -> f rest + +-- Check if the current context fulfills a StateDependency +fulfillsDependency ctx dep = + case dep of + DepGlobalValue name val -> (== (val, GlobalVar)) <$> peek ctx name + DepLocalValue name val -> (== (val, LocalVar)) <$> peek ctx name + DepPrefixValue name val -> (== (val, PrefixVar)) <$> peek ctx name + DepFunction name val -> (== val) <$> peekFunc ctx name + DepIsRecursive node val -> return $ val == any (\f -> entryPoint f == node) (cStack ctx) + -- _ -> error $ "Unknown dep " ++ show dep + where + peek = peekStack getVariableWithScope (unknownVariableValue, GlobalVar) + peekFunc = peekStack (\state name -> vmLookup name $ sFunctionTargets state) unknownFunctionValue + +-- Check if the current context fulfills all StateDependencies +fulfillsDependencies ctx deps = + f $ S.toList deps + where + f [] = return True + f (dep:rest) = do + res <- fulfillsDependency ctx dep + if res + then f rest + else return False + +-- Create a brand new Ctx given a Control Flow Graph (CFG) +newCtx g = do + c <- newSTRef 1 + input <- newSTRef undefined + output <- newSTRef undefined + node <- newSTRef undefined + cache <- newSTRef M.empty + invocations <- newSTRef M.empty + return $ Ctx { + cCounter = c, + cInput = input, + cOutput = output, + cNode = node, + cCache = cache, + cStack = [], + cInvocations = invocations, + cGraph = g + } + +-- The next incrementing version for VersionedMaps +nextVersion ctx = do + let ctr = cCounter ctx + n <- readSTRef ctr + writeSTRef ctr $! n+1 + return n + +-- Create a new StackEntry +newStackEntry ctx point = do + deps <- newSTRef S.empty + state <- readSTRef $ cOutput ctx + callsite <- readSTRef $ cNode ctx + return $ StackEntry { + entryPoint = point, + callSite = callsite, + dependencies = deps, + stackState = state + } + +-- Call a function with a new stack entry on the stack +withNewStackFrame ctx node f = do + newEntry <- newStackEntry ctx node + newInput <- newSTRef newInternalState + newOutput <- newSTRef newInternalState + newNode <- newSTRef node + let newCtx = ctx { + cInput = newInput, + cOutput = newOutput, + cNode = newNode, + cStack = newEntry : cStack ctx + } + x <- f newCtx + + {- + deps <- readSTRef $ dependencies newEntry + selfcheck <- fulfillsDependencies newCtx deps + unless selfcheck $ error $ pleaseReport $ "Unmet stack dependencies on " ++ show (node, deps) + -} + + return (x, newEntry) + +-- Check if invoking this function would be a recursive loop +-- (i.e. we already have the function on the stack) +wouldBeRecursive ctx node = f (cStack ctx) + where + f [] = return False + f (s:rest) = do + res <- + if entryPoint s == node + then return True + else f rest + modifySTRef (dependencies s) $ S.insert $ DepIsRecursive node res + return res + +-- The main DFA 'transfer' function, applying the effects of a node to the output state +transfer ctx label = + --traceShow ("Transferring", label) $ + case label of + CFStructuralNode -> return () + CFEntryPoint _ -> return () + CFImpliedExit -> return () + CFResolvedExit {} -> return () + + CFExecuteCommand cmd -> transferCommand ctx cmd + CFExecuteSubshell reason entry exit -> transferSubshell ctx reason entry exit + CFApplyEffects effects -> mapM_ (\(IdTagged _ f) -> transferEffect ctx f) effects + + CFUnresolvedExit -> patchOutputM ctx unreachableState + CFUnreachable -> patchOutputM ctx unreachableState + + -- TODO + CFSetBackgroundPid _ -> return () + CFSetExitCode _ -> return () + CFDropPrefixAssignments {} -> + modifySTRef (cOutput ctx) $ \c -> modified c { sPrefixValues = vmEmpty } +-- _ -> error $ "Unknown " ++ show label + + +-- Transfer the effects of a subshell invocation. This is similar to a function call +-- to allow easily discarding the effects (otherwise the InternalState would have +-- to represent subshell depth, while this way it can simply use the function stack). +transferSubshell ctx reason entry exit = do + let cout = cOutput ctx + initial <- readSTRef cout + runCached ctx entry (f entry exit) + -- Clear subshell changes. TODO: track this to warn about modifications. + writeSTRef cout initial + where + f entry exit ctx = do + (states, frame) <- withNewStackFrame ctx entry (flip dataflow $ entry) + let (_, res) = fromMaybe (error $ pleaseReport "Subshell has no exit") $ M.lookup exit states + deps <- readSTRef $ dependencies frame + registerFlowResult ctx entry states deps + return (deps, res) + +-- Transfer the effects of executing a command, i.e. the merged union of all possible function definitions. +transferCommand ctx Nothing = return () +transferCommand ctx (Just name) = do + targets <- readFunction ctx name + --traceShowM ("Transferring ",name,targets) + transferMultiple ctx $ map (flip transferFunctionValue) $ S.toList targets + +-- Transfer a set of function definitions and merge the output states. +transferMultiple ctx funcs = do +-- traceShowM ("Transferring set of ", length funcs) + original <- readSTRef out + branches <- mapM (apply ctx original) funcs + merged <- mergeStates ctx original branches + let patched = patchState original merged + writeSTRef out patched + where + out = cOutput ctx + apply ctx original f = do + writeSTRef out original + f ctx + readSTRef out + +-- Transfer the effects of a single function definition. +transferFunctionValue ctx funcVal = + case funcVal of + FunctionUnknown -> return () + FunctionDefinition name entry exit -> do + isRecursive <- wouldBeRecursive ctx entry + if isRecursive + then return () -- TODO: Find a better strategy for recursion + else runCached ctx entry (f name entry exit) + where + f name entry exit ctx = do + (states, frame) <- withNewStackFrame ctx entry (flip dataflow $ entry) + deps <- readSTRef $ dependencies frame + let res = + case M.lookup exit states of + Just (input, output) -> do + -- Discard local variables. TODO: track&retain variables declared local in previous scopes? + modified output { sLocalValues = vmEmpty } + Nothing -> do + -- e.g. f() { exit; } + unreachableState + registerFlowResult ctx entry states deps + return (deps, res) + + +-- Register/save the result of a dataflow of a function. +-- At the end, all the different values from different flows are merged together. +registerFlowResult ctx entry states deps = do + -- This function is called in the context of a CFExecuteCommand and not its invoked function, + -- so manually add the current node to the stack. + current <- readSTRef $ cNode ctx + let parents = map callSite $ cStack ctx + -- A unique path to this flow context. The specific value doesn't matter, as long as it's + -- unique per invocation of the function. This is required so that 'x=1; f; x=2; f' won't + -- overwrite each other. + let path = entry : current : parents + modifySTRef (cInvocations ctx) $ M.insert path (deps, states) + + +-- Look up a node in the cache and see if the dependencies of any entries are matched. +-- In that case, reuse the previous result instead of doing a new data flow. +runCached :: forall s. Ctx s -> Node -> (Ctx s -> ST s (S.Set StateDependency, InternalState)) -> ST s () +runCached ctx node f = do + cache <- getCache ctx node + case cache of + Just v -> do + -- traceShowM $ ("Running cached", node) + patchOutputM ctx v + Nothing -> do + -- traceShowM $ ("Cache failed", node) + (deps, diff) <- f ctx + modifySTRef (cCache ctx) (M.insertWith (\_ old -> (deps, diff):(take cacheEntries old)) node [(deps,diff)]) + -- traceShowM $ ("Recomputed cache for", node, deps) + patchOutputM ctx diff + +-- Get a cached version whose dependencies are currently fulfilled, if any. +getCache :: forall s. Ctx s -> Node -> ST s (Maybe InternalState) +getCache ctx node = do + cache <- readSTRef $ cCache ctx + -- traceShowM $ ("Cache for", node, "length", length $ M.findWithDefault [] node cache, M.lookup node cache) + f $ M.findWithDefault [] node cache + where + f [] = return Nothing + f ((deps, value):rest) = do + match <- fulfillsDependencies ctx deps + if match + then return $ Just value + else f rest + +-- Transfer a single CFEffect to the output state. +transferEffect ctx effect = + case effect of + CFReadVariable name -> do + void $ readVariable ctx name + CFWriteVariable name value -> do + val <- cfValueToVariableValue ctx value + writeVariable ctx name val + CFWriteGlobal name value -> do + val <- cfValueToVariableValue ctx value + writeGlobal ctx name val + CFWriteLocal name value -> do + val <- cfValueToVariableValue ctx value + writeLocal ctx name val + CFWritePrefix name value -> do + val <- cfValueToVariableValue ctx value + writePrefix ctx name val + CFUndefineVariable name -> undefineVariable ctx name + CFUndefineFunction name -> undefineFunction ctx name + CFUndefine name -> do + -- This should really just unset one or the other + undefineVariable ctx name + undefineFunction ctx name + CFDefineFunction name id entry exit -> + writeFunction ctx name $ FunctionDefinition name entry exit + + -- TODO + CFUndefineNameref name -> undefineVariable ctx name + CFHintArray name -> return () + CFHintDefined name -> return () + CFModifyProps {} -> return () +-- _ -> error $ "Unknown effect " ++ show effect + + + +-- Transfer the CFG's idea of a value into our VariableState +cfValueToVariableValue ctx val = + case val of + CFValueArray -> return unknownVariableValue -- TODO: Track array status + CFValueComputed _ parts -> foldM f emptyVariableValue parts + CFValueInteger -> return unknownIntegerValue + CFValueString -> return unknownVariableValue + CFValueUninitialized -> return emptyVariableValue +-- _ -> error $ "Unknown value: " ++ show val + where + f val part = do + next <- computeValue ctx part + return $ val `appendVariableValue` next + +-- A value can be computed from 0 or more parts, such as x="literal$y$z" +computeValue ctx part = + case part of + CFStringLiteral str -> return $ literalToVariableValue str + CFStringInteger -> return unknownIntegerValue + CFStringUnknown -> return unknownVariableValue + CFStringVariable name -> readVariable ctx name + +-- Append two VariableValues as if with z="$x$y" +appendVariableValue :: VariableValue -> VariableValue -> VariableValue +appendVariableValue a b = + VariableValue { + literalValue = liftM2 (++) (literalValue a) (literalValue b), + spaceStatus = appendSpaceStatus (spaceStatus a) (spaceStatus b) + } + +appendSpaceStatus a b = + case (a,b) of + (SpaceStatusEmpty, _) -> b + (_, SpaceStatusEmpty) -> a + (SpaceStatusClean, SpaceStatusClean) -> a + _ ->SpaceStatusDirty + +unknownIntegerValue = VariableValue { + literalValue = Nothing, + spaceStatus = SpaceStatusClean +} + +literalToVariableValue str = VariableValue { + literalValue = Just str, + spaceStatus = literalToSpaceStatus str +} + +withoutChanges ctx f = do + let inp = cInput ctx + let out = cOutput ctx + prevInput <- readSTRef inp + prevOutput <- readSTRef out + res <- f + writeSTRef inp prevInput + writeSTRef out prevOutput + return res + +-- Get the SpaceStatus for a literal string, i.e. if it needs quoting +literalToSpaceStatus str = + case str of + "" -> SpaceStatusEmpty + _ | all (`notElem` " \t\n*?[") str -> SpaceStatusClean + _ -> SpaceStatusDirty + +type StateMap = M.Map Node (InternalState, InternalState) + +-- Classic, iterative Data Flow Analysis. See Wikipedia for a description of the process. +dataflow :: forall s. Ctx s -> Node -> ST s StateMap +dataflow ctx entry = do + pending <- newSTRef $ S.singleton entry + states <- newSTRef $ M.empty + -- Should probably be done via a stack frame instead + withoutChanges ctx $ + f iterationCount pending states + readSTRef states + where + graph = cGraph ctx + f 0 _ _ = error $ pleaseReport "DFA did not reach fix point" + f n pending states = do + ps <- readSTRef pending + if S.null ps + then return () + else do + let (next, rest) = S.deleteFindMin ps + nexts <- process states next + writeSTRef pending $ foldl (flip S.insert) rest nexts + f (n-1) pending states + + process states node = do + stateMap <- readSTRef states + let inputs = filter (\c -> sIsReachable c /= Just False) $ mapMaybe (\c -> fmap snd $ M.lookup c stateMap) incoming + input <- + case incoming of + [] -> return newInternalState + _ -> + case inputs of + [] -> return unreachableState + (x:rest) -> foldM (mergeState ctx) x rest + writeSTRef (cInput ctx) $ input + writeSTRef (cOutput ctx) $ input + writeSTRef (cNode ctx) $ node + transfer ctx label + newOutput <- readSTRef $ cOutput ctx + result <- + if is2plus outgoing + then + -- Version the state because we split and will probably merge later + versionState ctx newOutput + else return newOutput + writeSTRef states $ M.insert node (input, result) stateMap + case M.lookup node stateMap of + Nothing -> return outgoing + Just (oldInput, oldOutput) -> + if oldOutput == result + then return [] + else return outgoing + where + (incomingL, _, label, outgoingL) = context graph $ node + incoming = map snd $ filter isRegular $ incomingL + outgoing = map snd outgoingL + isRegular = ((== CFEFlow) . fst) + +runRoot ctx entry exit = do + let env = createEnvironmentState + writeSTRef (cInput ctx) $ env + writeSTRef (cOutput ctx) $ env + writeSTRef (cNode ctx) $ entry + (states, frame) <- withNewStackFrame ctx entry $ \c -> dataflow c entry + deps <- readSTRef $ dependencies frame + registerFlowResult ctx entry states deps + -- Return the final state, used to invoke functions that were declared but not invoked + return $ snd $ fromMaybe (error $ pleaseReport "Missing exit state") $ M.lookup exit states + + +analyzeControlFlow :: CFGParameters -> Token -> CFGAnalysis +analyzeControlFlow params t = + let + cfg = buildGraph params t + (entry, exit) = M.findWithDefault (error $ pleaseReport "Missing root") (getId t) (cfIdToNode cfg) + in + runST $ f cfg entry exit + where + f cfg entry exit = do + ctx <- newCtx $ cfGraph cfg + -- Do a dataflow analysis starting on the root node + exitState <- runRoot ctx entry exit + + -- All nodes we've touched + invocations <- readSTRef $ cInvocations ctx + let invokedNodes = M.fromDistinctAscList $ map (\c -> (c, ())) $ S.toList $ M.keysSet $ groupByNode $ M.map snd invocations + + -- Invoke all functions that were declared but not invoked + -- This is so that we still get warnings for dead code + -- (it's probably not actually dead, just used by a script that sources ours) + let declaredFunctions = getFunctionTargets exitState + let uninvoked = M.difference declaredFunctions invokedNodes + analyzeStragglers ctx exitState uninvoked + + -- Now round up all the states from all data flows + -- (FIXME: this excludes functions that were defined in straggling functions) + invocations <- readSTRef $ cInvocations ctx + invokedStates <- flattenByNode ctx $ groupByNode $ M.map addDeps invocations + + -- Fill in the map with unreachable states for anything we didn't get to + let baseStates = M.fromDistinctAscList $ map (\c -> (c, (unreachableState, unreachableState))) $ uncurry enumFromTo $ nodeRange $ cfGraph cfg + let allStates = M.unionWith (flip const) baseStates invokedStates + + -- Convert to external states + let nodeToData = M.map (\(a,b) -> (internalToExternal a, internalToExternal b)) allStates + + return $ nodeToData `deepseq` CFGAnalysis { + graph = cfGraph cfg, + tokenToNode = cfIdToNode cfg, + nodeToData = nodeToData + } + + + -- Include the dependencies in the state of each function, e.g. if it depends on `x=foo` then add that. + addDeps :: (S.Set StateDependency, M.Map Node (InternalState, InternalState)) -> M.Map Node (InternalState, InternalState) + addDeps (deps, m) = let base = depsToState deps in M.map (\(a,b) -> (base `patchState` a, base `patchState` b)) m + + -- Collect all the states that each node has resulted in. + groupByNode :: forall k v. M.Map k (M.Map Node v) -> M.Map Node [v] + groupByNode pathMap = M.fromListWith (++) $ map (\(k,v) -> (k,[v])) $ concatMap M.toList $ M.elems pathMap + + -- Merge all the pre/post states for each node. This would have been a foldM if Map had one. + flattenByNode ctx m = M.fromDistinctAscList <$> (mapM (mergePair ctx) $ M.toList m) + + mergeAllStates ctx pairs = + let + (pres, posts) = unzip pairs + in do + pre <- mergeStates ctx (error $ pleaseReport "Null node states") pres + post <- mergeStates ctx (error $ pleaseReport "Null node states") posts + return (pre, post) + + mergePair ctx (node, list) = do + merged <- mergeAllStates ctx list + return (node, merged) + + -- Get the all the functions defined in an InternalState + getFunctionTargets :: InternalState -> M.Map Node FunctionDefinition + getFunctionTargets state = + let + declaredFuncs = S.unions $ mapStorage $ sFunctionTargets state + getFunc d = + case d of + FunctionDefinition _ entry _ -> Just (entry, d) + _ -> Nothing + funcs = mapMaybe getFunc $ S.toList declaredFuncs + in + M.fromList funcs + + +analyzeStragglers ctx state stragglers = do + mapM_ f $ M.elems stragglers + where + f def@(FunctionDefinition name entry exit) = do + writeSTRef (cInput ctx) state + writeSTRef (cOutput ctx) state + writeSTRef (cNode ctx) entry + transferFunctionValue ctx def + + +return [] +runTests = $quickCheckAll diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index e65dc68..cac06bc 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -30,6 +30,7 @@ import ShellCheck.AnalyzerLib import ShellCheck.Data import ShellCheck.Interface import ShellCheck.Parser +import ShellCheck.Prelude import ShellCheck.Regex import Control.Monad diff --git a/src/ShellCheck/Checks/ControlFlow.hs b/src/ShellCheck/Checks/ControlFlow.hs new file mode 100644 index 0000000..9b7635e --- /dev/null +++ b/src/ShellCheck/Checks/ControlFlow.hs @@ -0,0 +1,101 @@ +{- + Copyright 2022 Vidar Holen + + This file is part of ShellCheck. + https://www.shellcheck.net + + ShellCheck is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + ShellCheck is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +-} +{-# LANGUAGE TemplateHaskell #-} + +-- Checks that run on the Control Flow Graph (as opposed to the AST) +-- This is scaffolding for a work in progress. + +module ShellCheck.Checks.ControlFlow (checker, optionalChecks, ShellCheck.Checks.ControlFlow.runTests) where + +import ShellCheck.AST +import ShellCheck.ASTLib +import ShellCheck.CFG hiding (cfgAnalysis) +import ShellCheck.CFGAnalysis +import ShellCheck.AnalyzerLib +import ShellCheck.Data +import ShellCheck.Interface + +import Control.Monad +import Control.Monad.Reader +import Data.Graph.Inductive.Graph +import qualified Data.Map as M +import qualified Data.Set as S +import Data.List +import Data.Maybe + +import Test.QuickCheck.All (forAllProperties) +import Test.QuickCheck.Test (quickCheckWithResult, stdArgs, maxSuccess) + + +optionalChecks :: [CheckDescription] +optionalChecks = [] + +-- A check that runs on the entire graph +type ControlFlowCheck = Analysis +-- A check invoked once per node, with its (pre,post) data +type ControlFlowNodeCheck = LNode CFNode -> (ProgramState, ProgramState) -> Analysis +-- A check invoked once per effect, with its node's (pre,post) data +type ControlFlowEffectCheck = IdTagged CFEffect -> Node -> (ProgramState, ProgramState) -> Analysis + + +checker :: AnalysisSpec -> Parameters -> Checker +checker spec params = Checker { + perScript = const $ sequence_ controlFlowChecks, + perToken = const $ return () +} + +controlFlowChecks :: [ControlFlowCheck] +controlFlowChecks = [ + runNodeChecks controlFlowNodeChecks + ] + +controlFlowNodeChecks :: [ControlFlowNodeCheck] +controlFlowNodeChecks = [ + runEffectChecks controlFlowEffectChecks + ] + +controlFlowEffectChecks :: [ControlFlowEffectCheck] +controlFlowEffectChecks = [ + ] + +runNodeChecks :: [ControlFlowNodeCheck] -> ControlFlowCheck +runNodeChecks perNode = do + cfg <- asks cfgAnalysis + runOnAll cfg + where + getData datas n@(node, label) = do + (pre, post) <- M.lookup node datas + return (n, (pre, post)) + + runOn :: (LNode CFNode, (ProgramState, ProgramState)) -> Analysis + runOn (node, prepost) = mapM_ (\c -> c node prepost) perNode + runOnAll cfg = mapM_ runOn $ mapMaybe (getData $ nodeToData cfg) $ labNodes (graph cfg) + +runEffectChecks :: [ControlFlowEffectCheck] -> ControlFlowNodeCheck +runEffectChecks list = checkNode + where + checkNode (node, label) prepost = + case label of + CFApplyEffects effects -> mapM_ (\effect -> mapM_ (\c -> c effect node prepost) list) effects + _ -> return () + + +return [] +runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 22a6a5f..9ad17f5 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -25,6 +25,7 @@ import ShellCheck.AST import ShellCheck.ASTLib import ShellCheck.AnalyzerLib import ShellCheck.Interface +import ShellCheck.Prelude import ShellCheck.Regex import Control.Monad diff --git a/src/ShellCheck/Data.hs b/src/ShellCheck/Data.hs index e22b424..fb82ca8 100644 --- a/src/ShellCheck/Data.hs +++ b/src/ShellCheck/Data.hs @@ -2,9 +2,27 @@ module ShellCheck.Data where import ShellCheck.Interface import Data.Version (showVersion) -import Paths_ShellCheck (version) -shellcheckVersion = showVersion version -- VERSIONSTRING + +{- +If you are here because you saw an error about Paths_ShellCheck in this file, +simply comment out the import below and define the version as a constant string. + +Instead of: + + import Paths_ShellCheck (version) + shellcheckVersion = showVersion version + +Use: + + -- import Paths_ShellCheck (version) + shellcheckVersion = "kludge" + +-} + +import Paths_ShellCheck (version) +shellcheckVersion = showVersion version -- VERSIONSTRING + internalVariables = [ -- Generic @@ -43,9 +61,12 @@ internalVariables = [ "flags_error", "flags_return" ] -specialVariablesWithoutSpaces = [ - "$", "-", "?", "!", "#" +specialIntegerVariables = [ + "$", "?", "!", "#" ] + +specialVariablesWithoutSpaces = "-" : specialIntegerVariables + variablesWithoutSpaces = specialVariablesWithoutSpaces ++ [ "BASHPID", "BASH_ARGC", "BASH_LINENO", "BASH_SUBSHELL", "EUID", "LINENO", "OPTIND", "PPID", "RANDOM", "SECONDS", "SHELLOPTS", "SHLVL", "UID", diff --git a/src/ShellCheck/Debug.hs b/src/ShellCheck/Debug.hs new file mode 100644 index 0000000..c991308 --- /dev/null +++ b/src/ShellCheck/Debug.hs @@ -0,0 +1,313 @@ +{- + +This file contains useful functions for debugging and developing ShellCheck. + +To invoke them interactively, run: + + cabal repl + +At the ghci prompt, enter: + + :load ShellCheck.Debug + +You can now invoke the functions. Here are some examples: + + shellcheckString "echo $1" + stringToAst "(( x+1 ))" + stringToCfg "if foo; then bar; else baz; fi" + writeFile "/tmp/test.dot" $ stringToCfgViz "while foo; do bar; done" + +The latter file can be rendered to png with GraphViz: + + dot -Tpng /tmp/test.dot > /tmp/test.png + +To run all unit tests in a module: + + ShellCheck.Parser.runTests + ShellCheck.Analytics.runTests + +To run a specific test: + + :load ShellCheck.Analytics + prop_checkUuoc3 + +If you make code changes, reload in seconds at any time with: + + :r + +=========================================================================== + +Crash course in printf debugging in Haskell: + + import Debug.Trace + + greet 0 = return () + -- Print when a function is invoked + greet n | trace ("calling greet " ++ show n) False = undefined + greet n = do + putStrLn "Enter name" + name <- getLine + -- Print at some point in any monadic function + traceM $ "user entered " ++ name + putStrLn $ "Hello " ++ name + -- Print a value before passing it on + greet $ traceShowId (n - 1) + + +=========================================================================== + +If you want to invoke `ghci` directly, such as on `shellcheck.hs`, to +debug all of ShellCheck including I/O, you may see an error like this: + + src/ShellCheck/Data.hs:5:1: error: + Could not load module ‘Paths_ShellCheck’ + it is a hidden module in the package ‘ShellCheck-0.8.0’ + +This can easily be circumvented by running `./setgitversion` or manually +editing src/ShellCheck/Data.hs to replace the auto-deduced version number +with a constant string as indicated. + +Afterwards, you can run the ShellCheck tool, as if from the shell, with: + + $ ghci shellcheck.hs + ghci> runMain ["-x", "file.sh"] + +-} + +module ShellCheck.Debug () where + +import ShellCheck.Analyzer +import ShellCheck.AST +import ShellCheck.CFG +import ShellCheck.Checker +import ShellCheck.CFGAnalysis as CF +import ShellCheck.Interface +import ShellCheck.Parser +import ShellCheck.Prelude + +import Control.Monad +import Control.Monad.Identity +import Control.Monad.RWS +import Control.Monad.Writer +import Data.Graph.Inductive.Graph as G +import Data.List +import Data.Maybe +import qualified Data.Map as M +import qualified Data.Set as S + + +-- Run all of ShellCheck (minus output formatters) +shellcheckString :: String -> CheckResult +shellcheckString scriptString = + runIdentity $ checkScript dummySystemInterface checkSpec + where + checkSpec :: CheckSpec + checkSpec = emptyCheckSpec { + csScript = scriptString + } + +dummySystemInterface :: SystemInterface Identity +dummySystemInterface = mockedSystemInterface [ + -- A tiny, fake filesystem for sourced files + ("lib/mylib1.sh", "foo=$(cat $1 | wc -l)"), + ("lib/mylib2.sh", "bar=42") + ] + +-- Parameters used when generating Control Flow Graphs +cfgParams :: CFGParameters +cfgParams = CFGParameters { + cfLastpipe = False, + cfPipefail = False +} + +-- An example script to play with +exampleScript :: String +exampleScript = unlines [ + "#!/bin/sh", + "count=0", + "for file in *", + "do", + " (( count++ ))", + "done", + "echo $count" + ] + +-- Parse the script string into ShellCheck's ParseResult +parseScriptString :: String -> ParseResult +parseScriptString scriptString = + runIdentity $ parseScript dummySystemInterface parseSpec + where + parseSpec :: ParseSpec + parseSpec = newParseSpec { + psFilename = "myscript", + psScript = scriptString + } + + +-- Parse the script string into an Abstract Syntax Tree +stringToAst :: String -> Token +stringToAst scriptString = + case maybeRoot of + Just root -> root + Nothing -> error $ "Script failed to parse: " ++ show parserWarnings + where + parseResult :: ParseResult + parseResult = parseScriptString scriptString + + maybeRoot :: Maybe Token + maybeRoot = prRoot parseResult + + parserWarnings :: [PositionedComment] + parserWarnings = prComments parseResult + + +astToCfgResult :: Token -> CFGResult +astToCfgResult = buildGraph cfgParams + +astToDfa :: Token -> CFGAnalysis +astToDfa = analyzeControlFlow cfgParams + +astToCfg :: Token -> CFGraph +astToCfg = cfGraph . astToCfgResult + +stringToCfg :: String -> CFGraph +stringToCfg = astToCfg . stringToAst + +stringToDfa :: String -> CFGAnalysis +stringToDfa = astToDfa . stringToAst + +cfgToGraphViz :: CFGraph -> String +cfgToGraphViz = cfgToGraphVizWith show + +stringToCfgViz :: String -> String +stringToCfgViz = cfgToGraphViz . stringToCfg + +stringToDfaViz :: String -> String +stringToDfaViz = dfaToGraphViz . stringToDfa + +-- Dump a Control Flow Graph as GraphViz with extended information +stringToDetailedCfgViz :: String -> String +stringToDetailedCfgViz scriptString = cfgToGraphVizWith nodeLabel graph + where + ast :: Token + ast = stringToAst scriptString + + cfgResult :: CFGResult + cfgResult = astToCfgResult ast + + graph :: CFGraph + graph = cfGraph cfgResult + + idToToken :: M.Map Id Token + idToToken = M.fromList $ execWriter $ doAnalysis (\c -> tell [(getId c, c)]) ast + + idToNode :: M.Map Id (Node, Node) + idToNode = cfIdToNode cfgResult + + nodeToStartIds :: M.Map Node (S.Set Id) + nodeToStartIds = + M.fromListWith S.union $ + map (\(id, (start, _)) -> (start, S.singleton id)) $ + M.toList idToNode + + nodeToEndIds :: M.Map Node (S.Set Id) + nodeToEndIds = + M.fromListWith S.union $ + map (\(id, (_, end)) -> (end, S.singleton id)) $ + M.toList idToNode + + formatId :: Id -> String + formatId id = fromMaybe ("Unknown " ++ show id) $ do + (OuterToken _ token) <- M.lookup id idToToken + firstWord <- words (show token) !!! 0 + -- Strip off "Inner_" + (_ : tokenName) <- return $ dropWhile (/= '_') firstWord + return $ tokenName ++ " " ++ show id + + formatGroup :: S.Set Id -> String + formatGroup set = intercalate ", " $ map formatId $ S.toList set + + nodeLabel (node, label) = unlines [ + show node ++ ". " ++ show label, + "Begin: " ++ formatGroup (M.findWithDefault S.empty node nodeToStartIds), + "End: " ++ formatGroup (M.findWithDefault S.empty node nodeToEndIds) + ] + + +-- Dump a Control Flow Graph with Data Flow Analysis as GraphViz +dfaToGraphViz :: CF.CFGAnalysis -> String +dfaToGraphViz analysis = cfgToGraphVizWith label $ CF.graph analysis + where + label (node, label) = + let + desc = show node ++ ". " ++ show label + in + fromMaybe ("No DFA available\n\n" ++ desc) $ do + (pre, post) <- M.lookup node $ CF.nodeToData analysis + return $ unlines [ + "Precondition: " ++ show pre, + "", + desc, + "", + "Postcondition: " ++ show post + ] + + +-- Dump an Control Flow Graph to GraphViz with a given node formatter +cfgToGraphVizWith :: (LNode CFNode -> String) -> CFGraph -> String +cfgToGraphVizWith nodeLabel graph = concat [ + "digraph {\n", + concatMap dumpNode (labNodes graph), + concatMap dumpLink (labEdges graph), + tagVizEntries graph, + "}\n" + ] + where + dumpNode l@(node, label) = show node ++ " [label=" ++ quoteViz (nodeLabel l) ++ "]\n" + dumpLink (from, to, typ) = show from ++ " -> " ++ show to ++ " [style=" ++ quoteViz (edgeStyle typ) ++ "]\n" + edgeStyle CFEFlow = "solid" + edgeStyle CFEExit = "bold" + edgeStyle CFEFalseFlow = "dotted" + +quoteViz str = "\"" ++ escapeViz str ++ "\"" +escapeViz [] = [] +escapeViz (c:rest) = + case c of + '\"' -> '\\' : '\"' : escapeViz rest + '\n' -> '\\' : 'l' : escapeViz rest + '\\' -> '\\' : '\\' : escapeViz rest + _ -> c : escapeViz rest + + +-- Dump an Abstract Syntax Tree (or branch thereof) to GraphViz format +astToGraphViz :: Token -> String +astToGraphViz token = concat [ + "digraph {\n", + formatTree token, + "}\n" + ] + where + formatTree :: Token -> String + formatTree t = snd $ execRWS (doStackAnalysis push pop t) () [] + + push :: Token -> RWS () String [Int] () + push (OuterToken (Id n) inner) = do + stack <- get + put (n : stack) + case stack of + [] -> return () + (top:_) -> tell $ show top ++ " -> " ++ show n ++ "\n" + tell $ show n ++ " [label=" ++ quoteViz (show n ++ ": " ++ take 32 (show inner)) ++ "]\n" + + pop :: Token -> RWS () String [Int] () + pop _ = modify tail + + +-- For each entry point, set the rank so that they'll align in the graph +tagVizEntries :: CFGraph -> String +tagVizEntries graph = "{ rank=same " ++ rank ++ " }" + where + entries = mapMaybe find $ labNodes graph + find (node, CFEntryPoint name) = return (node, name) + find _ = Nothing + rank = unwords $ map (\(c, _) -> show c) entries diff --git a/src/ShellCheck/Fixer.hs b/src/ShellCheck/Fixer.hs index 1409b24..2376842 100644 --- a/src/ShellCheck/Fixer.hs +++ b/src/ShellCheck/Fixer.hs @@ -22,6 +22,7 @@ module ShellCheck.Fixer (applyFix, removeTabStops, mapPositions, Ranged(..), runTests) where import ShellCheck.Interface +import ShellCheck.Prelude import Control.Monad.State import Data.Array import Data.List @@ -228,7 +229,7 @@ applyReplacement2 rep string = do let (l1, l2) = tmap posLine originalPos in when (l1 /= 1 || l2 /= 1) $ - error "ShellCheck internal error, please report: bad cross-line fix" + error $ pleaseReport "bad cross-line fix" let replacer = repString rep let shift = (length replacer) - (oldEnd - oldStart) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 3958406..9f9241c 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -27,6 +27,7 @@ import ShellCheck.AST import ShellCheck.ASTLib hiding (runTests) import ShellCheck.Data import ShellCheck.Interface +import ShellCheck.Prelude import Control.Applicative ((<*), (*>)) import Control.Monad @@ -210,7 +211,7 @@ getNextIdSpanningTokenList list = -- Get the span covered by an id getSpanForId :: Monad m => Id -> SCParser m (SourcePos, SourcePos) getSpanForId id = - Map.findWithDefault (error "Internal error: no position for id. Please report!") id <$> + Map.findWithDefault (error $ pleaseReport "no parser span for id") id <$> getMap -- Create a new id with the same span as an existing one @@ -1918,7 +1919,7 @@ readPendingHereDocs = do -- The end token is just a prefix skipLine | hasTrailer -> - error "ShellCheck bug, please report (here doc trailer)." + error $ pleaseReport "unexpected heredoc trailer" -- The following cases assume no trailing text: | dashed == Undashed && (not $ null leadingSpace) -> do diff --git a/src/ShellCheck/Prelude.hs b/src/ShellCheck/Prelude.hs new file mode 100644 index 0000000..7e9011b --- /dev/null +++ b/src/ShellCheck/Prelude.hs @@ -0,0 +1,48 @@ +{- + Copyright 2022 Vidar Holen + + This file is part of ShellCheck. + https://www.shellcheck.net + + ShellCheck is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + ShellCheck is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +-} + +-- Generic basic utility functions +module ShellCheck.Prelude where + +-- Get element 0 or a default. Like `head` but safe. +headOrDefault _ (a:_) = a +headOrDefault def _ = def + +-- Get the last element or a default. Like `last` but safe. +lastOrDefault def [] = def +lastOrDefault _ list = last list + +--- Get element n of a list, or Nothing. Like `!!` but safe. +(!!!) list i = + case drop i list of + [] -> Nothing + (r:_) -> Just r + + +-- Like mconcat but for Semigroups +sconcat1 :: (Semigroup t) => [t] -> t +sconcat1 [x] = x +sconcat1 (x:xs) = x <> sconcat1 xs + +sconcatOrDefault def [] = def +sconcatOrDefault _ list = sconcat1 list + +-- For more actionable "impossible" errors +pleaseReport str = "ShellCheck internal error, please report: " ++ str diff --git a/test/shellcheck.hs b/test/shellcheck.hs index e463403..1a272af 100644 --- a/test/shellcheck.hs +++ b/test/shellcheck.hs @@ -5,8 +5,11 @@ import System.Exit import qualified ShellCheck.Analytics import qualified ShellCheck.AnalyzerLib import qualified ShellCheck.ASTLib +import qualified ShellCheck.CFG +import qualified ShellCheck.CFGAnalysis import qualified ShellCheck.Checker import qualified ShellCheck.Checks.Commands +import qualified ShellCheck.Checks.ControlFlow import qualified ShellCheck.Checks.Custom import qualified ShellCheck.Checks.ShellSupport import qualified ShellCheck.Fixer @@ -19,8 +22,11 @@ main = do ShellCheck.Analytics.runTests ,ShellCheck.AnalyzerLib.runTests ,ShellCheck.ASTLib.runTests + ,ShellCheck.CFG.runTests + ,ShellCheck.CFGAnalysis.runTests ,ShellCheck.Checker.runTests ,ShellCheck.Checks.Commands.runTests + ,ShellCheck.Checks.ControlFlow.runTests ,ShellCheck.Checks.Custom.runTests ,ShellCheck.Checks.ShellSupport.runTests ,ShellCheck.Fixer.runTests From 642ad8612597b28af4026bde289985583fddb69d Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 19 Jul 2022 14:33:00 -0700 Subject: [PATCH 543/763] Add SC2317 warning about unreachable commands --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 32 ++++++++++++++++++++++++-------- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4763ddd..6e1dba3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## Git ### Added - SC2316: Warn about 'local readonly foo' and similar (thanks, patrickxia!) +- SC2317: Warn about unreachable commands ### Fixed diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index bf5d179..e6c0160 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -202,6 +202,7 @@ nodeChecks = [ ,checkCommandWithTrailingSymbol ,checkUnquotedParameterExpansionPattern ,checkBatsTestDoesNotUseNegation + ,checkCommandIsUnreachable ] optionalChecks = map fst optionalTreeChecks @@ -4129,13 +4130,6 @@ checkAliasUsedInSameParsingUnit params root = checkUnit :: [Token] -> Writer [TokenComment] () checkUnit unit = evalStateT (mapM_ (doAnalysis findCommands) unit) (Map.empty) - isSourced t = - let - f (T_SourceCommand {}) = True - f _ = False - in - any f $ getPath (parentMap params) t - findCommands :: Token -> StateT (Map.Map String Token) (Writer [TokenComment]) () findCommands t = case t of T_SimpleCommand _ _ (cmd:args) -> @@ -4146,7 +4140,7 @@ checkAliasUsedInSameParsingUnit params root = cmd <- gets (Map.lookup name) case cmd of Just alias -> - unless (isSourced t || shouldIgnoreCode params 2262 alias) $ do + unless (isSourced params t || shouldIgnoreCode params 2262 alias) $ do warn (getId alias) 2262 "This alias can't be defined and used in the same parsing unit. Use a function instead." info (getId t) 2263 "Since they're in the same parsing unit, this command will not refer to the previously mentioned alias." _ -> return () @@ -4157,6 +4151,14 @@ checkAliasUsedInSameParsingUnit params root = when (isVariableName name && not (null value)) $ modify (Map.insertWith (\new old -> old) name arg) +isSourced params t = + let + f (T_SourceCommand {}) = True + f _ = False + in + any f $ getPath (parentMap params) t + + -- Like groupBy, but compares pairs of adjacent elements, rather than against the first of the span prop_groupByLink1 = groupByLink (\a b -> a+1 == b) [1,2,3,2,3,7,8,9] == [[1,2,3], [2,3], [7,8,9]] prop_groupByLink2 = groupByLink (==) ([] :: [()]) == [] @@ -4910,5 +4912,19 @@ checkBatsTestDoesNotUseNegation params t = x:rest -> isLastOf t rest [] -> False + +prop_checkCommandIsUnreachable1 = verify checkCommandIsUnreachable "foo; bar; exit; baz" +prop_checkCommandIsUnreachable2 = verify checkCommandIsUnreachable "die() { exit; }; foo; bar; die; baz" +prop_checkCommandIsUnreachable3 = verifyNot checkCommandIsUnreachable "foo; bar || exit; baz" +checkCommandIsUnreachable params t = + case t of + T_Pipeline {} -> sequence_ $ do + state <- CF.getIncomingState (cfgAnalysis params) id + guard . not $ CF.stateIsReachable state + guard . not $ isSourced params t + return $ info id 2317 "Command appears to be unreachable. Check usage (or ignore if invoked indirectly)." + _ -> return () + where id = getId t + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) From da4885a71d08c76c6ff5d01012d135d54fa68f4e Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 19 Jul 2022 14:33:00 -0700 Subject: [PATCH 544/763] Use DFA for SC2086 --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 272 ++++++++++++++---------------------- 2 files changed, 104 insertions(+), 169 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e1dba3..66d5369 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - SC2317: Warn about unreachable commands ### Fixed +- SC2086: Now uses DFA to make more accurate predictions about values ### Changed - ShellCheck now has a Data Flow Analysis engine to make smarter decisions diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index e6c0160..0062879 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -54,7 +54,6 @@ treeChecks :: [Parameters -> Token -> [TokenComment]] treeChecks = [ nodeChecksToTreeCheck nodeChecks ,subshellAssignmentCheck - ,checkSpacefulness ,checkQuotesInLiterals ,checkShebangParameters ,checkFunctionsUsedExternally @@ -203,6 +202,7 @@ nodeChecks = [ ,checkUnquotedParameterExpansionPattern ,checkBatsTestDoesNotUseNegation ,checkCommandIsUnreachable + ,checkSpacefulnessCfg ] optionalChecks = map fst optionalTreeChecks @@ -221,7 +221,7 @@ optionalTreeChecks = [ cdDescription = "Suggest quoting variables without metacharacters", cdPositive = "var=hello; echo $var", cdNegative = "var=hello; echo \"$var\"" - }, checkVerboseSpacefulness) + }, nodeChecksToTreeCheck [checkVerboseSpacefulnessCfg]) ,(newCheckDescription { cdName = "avoid-nullary-conditions", @@ -2009,109 +2009,6 @@ doVariableFlowAnalysis readFunc writeFunc empty flow = evalState ( writeFunc base token name values doFlow _ = return [] ----- Check whether variables could have spaces/globs -prop_checkSpacefulness1 = verifyTree checkSpacefulness "a='cow moo'; echo $a" -prop_checkSpacefulness2 = verifyNotTree checkSpacefulness "a='cow moo'; [[ $a ]]" -prop_checkSpacefulness3 = verifyNotTree checkSpacefulness "a='cow*.mp3'; echo \"$a\"" -prop_checkSpacefulness4 = verifyTree checkSpacefulness "for f in *.mp3; do echo $f; done" -prop_checkSpacefulness4a= verifyNotTree checkSpacefulness "foo=3; foo=$(echo $foo)" -prop_checkSpacefulness5 = verifyTree checkSpacefulness "a='*'; b=$a; c=lol${b//foo/bar}; echo $c" -prop_checkSpacefulness6 = verifyTree checkSpacefulness "a=foo$(lol); echo $a" -prop_checkSpacefulness7 = verifyTree checkSpacefulness "a=foo\\ bar; rm $a" -prop_checkSpacefulness8 = verifyNotTree checkSpacefulness "a=foo\\ bar; a=foo; rm $a" -prop_checkSpacefulness10= verifyTree checkSpacefulness "rm $1" -prop_checkSpacefulness11= verifyTree checkSpacefulness "rm ${10//foo/bar}" -prop_checkSpacefulness12= verifyNotTree checkSpacefulness "(( $1 + 3 ))" -prop_checkSpacefulness13= verifyNotTree checkSpacefulness "if [[ $2 -gt 14 ]]; then true; fi" -prop_checkSpacefulness14= verifyNotTree checkSpacefulness "foo=$3 env" -prop_checkSpacefulness15= verifyNotTree checkSpacefulness "local foo=$1" -prop_checkSpacefulness16= verifyNotTree checkSpacefulness "declare foo=$1" -prop_checkSpacefulness17= verifyTree checkSpacefulness "echo foo=$1" -prop_checkSpacefulness18= verifyNotTree checkSpacefulness "$1 --flags" -prop_checkSpacefulness19= verifyTree checkSpacefulness "echo $PWD" -prop_checkSpacefulness20= verifyNotTree checkSpacefulness "n+='foo bar'" -prop_checkSpacefulness21= verifyNotTree checkSpacefulness "select foo in $bar; do true; done" -prop_checkSpacefulness22= verifyNotTree checkSpacefulness "echo $\"$1\"" -prop_checkSpacefulness23= verifyNotTree checkSpacefulness "a=(1); echo ${a[@]}" -prop_checkSpacefulness24= verifyTree checkSpacefulness "a='a b'; cat <<< $a" -prop_checkSpacefulness25= verifyTree checkSpacefulness "a='s/[0-9]//g'; sed $a" -prop_checkSpacefulness26= verifyTree checkSpacefulness "a='foo bar'; echo {1,2,$a}" -prop_checkSpacefulness27= verifyNotTree checkSpacefulness "echo ${a:+'foo'}" -prop_checkSpacefulness28= verifyNotTree checkSpacefulness "exec {n}>&1; echo $n" -prop_checkSpacefulness29= verifyNotTree checkSpacefulness "n=$(stuff); exec {n}>&-;" -prop_checkSpacefulness30= verifyTree checkSpacefulness "file='foo bar'; echo foo > $file;" -prop_checkSpacefulness31= verifyNotTree checkSpacefulness "echo \"`echo \\\"$1\\\"`\"" -prop_checkSpacefulness32= verifyNotTree checkSpacefulness "var=$1; [ -v var ]" -prop_checkSpacefulness33= verifyTree checkSpacefulness "for file; do echo $file; done" -prop_checkSpacefulness34= verifyTree checkSpacefulness "declare foo$n=$1" -prop_checkSpacefulness35= verifyNotTree checkSpacefulness "echo ${1+\"$1\"}" -prop_checkSpacefulness36= verifyNotTree checkSpacefulness "arg=$#; echo $arg" -prop_checkSpacefulness37= verifyNotTree checkSpacefulness "@test 'status' {\n [ $status -eq 0 ]\n}" -prop_checkSpacefulness37v = verifyTree checkVerboseSpacefulness "@test 'status' {\n [ $status -eq 0 ]\n}" -prop_checkSpacefulness38= verifyTree checkSpacefulness "a=; echo $a" -prop_checkSpacefulness39= verifyNotTree checkSpacefulness "a=''\"\"''; b=x$a; echo $b" -prop_checkSpacefulness40= verifyNotTree checkSpacefulness "a=$((x+1)); echo $a" -prop_checkSpacefulness41= verifyNotTree checkSpacefulness "exec $1 --flags" -prop_checkSpacefulness42= verifyNotTree checkSpacefulness "run $1 --flags" -prop_checkSpacefulness43= verifyNotTree checkSpacefulness "$foo=42" -prop_checkSpacefulness44= verifyTree checkSpacefulness "#!/bin/sh\nexport var=$value" -prop_checkSpacefulness45= verifyNotTree checkSpacefulness "wait -zzx -p foo; echo $foo" -prop_checkSpacefulness46= verifyNotTree checkSpacefulness "x=0; (( x += 1 )); echo $x" -prop_checkSpacefulness47= verifyNotTree checkSpacefulness "x=0; (( x-- )); echo $x" -prop_checkSpacefulness48= verifyNotTree checkSpacefulness "x=0; (( ++x )); echo $x" - -data SpaceStatus = SpaceSome | SpaceNone | SpaceEmpty deriving (Eq) -instance Semigroup SpaceStatus where - SpaceNone <> SpaceNone = SpaceNone - SpaceSome <> _ = SpaceSome - _ <> SpaceSome = SpaceSome - SpaceEmpty <> x = x - x <> SpaceEmpty = x -instance Monoid SpaceStatus where - mempty = SpaceEmpty - mappend = (<>) - --- This is slightly awkward because we want to support structured --- optional checks based on nearly the same logic -checkSpacefulness params = checkSpacefulness' onFind params - where - emit x = tell [x] - onFind spaces token _ = - when (spaces /= SpaceNone) $ - if isDefaultAssignment (parentMap params) token - then - emit $ makeComment InfoC (getId token) 2223 - "This default assignment may cause DoS due to globbing. Quote it." - else - unless (quotesMayConflictWithSC2281 params token) $ - emit $ makeCommentWithFix InfoC (getId token) 2086 - "Double quote to prevent globbing and word splitting." - (addDoubleQuotesAround params token) - - isDefaultAssignment parents token = - let modifier = getBracedModifier $ bracedString token in - any (`isPrefixOf` modifier) ["=", ":="] - && isParamTo parents ":" token - - -- Given a T_DollarBraced, return a simplified version of the string contents. - bracedString (T_DollarBraced _ _ l) = concat $ oversimplify l - bracedString _ = error "Internal shellcheck error, please report! (bracedString on non-variable)" - -prop_checkSpacefulness4v= verifyTree checkVerboseSpacefulness "foo=3; foo=$(echo $foo)" -prop_checkSpacefulness8v= verifyTree checkVerboseSpacefulness "a=foo\\ bar; a=foo; rm $a" -prop_checkSpacefulness28v = verifyTree checkVerboseSpacefulness "exec {n}>&1; echo $n" -prop_checkSpacefulness36v = verifyTree checkVerboseSpacefulness "arg=$#; echo $arg" -prop_checkSpacefulness44v = verifyNotTree checkVerboseSpacefulness "foo=3; $foo=4" -checkVerboseSpacefulness params = checkSpacefulness' onFind params - where - onFind spaces token name = - when (spaces == SpaceNone - && name `notElem` specialVariablesWithoutSpaces - && not (quotesMayConflictWithSC2281 params token)) $ - tell [makeCommentWithFix StyleC (getId token) 2248 - "Prefer double quoting even when variables don't contain special characters." - (addDoubleQuotesAround params token)] - -- Don't suggest quotes if this will instead be autocorrected -- from $foo=bar to foo=bar. This is not pretty but ok. quotesMayConflictWithSC2281 params t = @@ -2121,74 +2018,111 @@ quotesMayConflictWithSC2281 params t = _ -> False addDoubleQuotesAround params token = (surroundWith (getId token) params "\"") -checkSpacefulness' - :: (SpaceStatus -> Token -> String -> Writer [TokenComment] ()) -> - Parameters -> Token -> [TokenComment] -checkSpacefulness' onFind params t = - doVariableFlowAnalysis readF writeF (Map.fromList defaults) (variableFlow params) + +prop_checkSpacefulnessCfg1 = verify checkSpacefulnessCfg "a='cow moo'; echo $a" +prop_checkSpacefulnessCfg2 = verifyNot checkSpacefulnessCfg "a='cow moo'; [[ $a ]]" +prop_checkSpacefulnessCfg3 = verifyNot checkSpacefulnessCfg "a='cow*.mp3'; echo \"$a\"" +prop_checkSpacefulnessCfg4 = verify checkSpacefulnessCfg "for f in *.mp3; do echo $f; done" +prop_checkSpacefulnessCfg4a= verifyNot checkSpacefulnessCfg "foo=3; foo=$(echo $foo)" +prop_checkSpacefulnessCfg5 = verify checkSpacefulnessCfg "a='*'; b=$a; c=lol${b//foo/bar}; echo $c" +prop_checkSpacefulnessCfg6 = verify checkSpacefulnessCfg "a=foo$(lol); echo $a" +prop_checkSpacefulnessCfg7 = verify checkSpacefulnessCfg "a=foo\\ bar; rm $a" +prop_checkSpacefulnessCfg8 = verifyNot checkSpacefulnessCfg "a=foo\\ bar; a=foo; rm $a" +prop_checkSpacefulnessCfg10= verify checkSpacefulnessCfg "rm $1" +prop_checkSpacefulnessCfg11= verify checkSpacefulnessCfg "rm ${10//foo/bar}" +prop_checkSpacefulnessCfg12= verifyNot checkSpacefulnessCfg "(( $1 + 3 ))" +prop_checkSpacefulnessCfg13= verifyNot checkSpacefulnessCfg "if [[ $2 -gt 14 ]]; then true; fi" +prop_checkSpacefulnessCfg14= verifyNot checkSpacefulnessCfg "foo=$3 env" +prop_checkSpacefulnessCfg15= verifyNot checkSpacefulnessCfg "local foo=$1" +prop_checkSpacefulnessCfg16= verifyNot checkSpacefulnessCfg "declare foo=$1" +prop_checkSpacefulnessCfg17= verify checkSpacefulnessCfg "echo foo=$1" +prop_checkSpacefulnessCfg18= verifyNot checkSpacefulnessCfg "$1 --flags" +prop_checkSpacefulnessCfg19= verify checkSpacefulnessCfg "echo $PWD" +prop_checkSpacefulnessCfg20= verifyNot checkSpacefulnessCfg "n+='foo bar'" +prop_checkSpacefulnessCfg21= verifyNot checkSpacefulnessCfg "select foo in $bar; do true; done" +prop_checkSpacefulnessCfg22= verifyNot checkSpacefulnessCfg "echo $\"$1\"" +prop_checkSpacefulnessCfg23= verifyNot checkSpacefulnessCfg "a=(1); echo ${a[@]}" +prop_checkSpacefulnessCfg24= verify checkSpacefulnessCfg "a='a b'; cat <<< $a" +prop_checkSpacefulnessCfg25= verify checkSpacefulnessCfg "a='s/[0-9]//g'; sed $a" +prop_checkSpacefulnessCfg26= verify checkSpacefulnessCfg "a='foo bar'; echo {1,2,$a}" +prop_checkSpacefulnessCfg27= verifyNot checkSpacefulnessCfg "echo ${a:+'foo'}" +prop_checkSpacefulnessCfg28= verifyNot checkSpacefulnessCfg "exec {n}>&1; echo $n" +prop_checkSpacefulnessCfg29= verifyNot checkSpacefulnessCfg "n=$(stuff); exec {n}>&-;" +prop_checkSpacefulnessCfg30= verify checkSpacefulnessCfg "file='foo bar'; echo foo > $file;" +prop_checkSpacefulnessCfg31= verifyNot checkSpacefulnessCfg "echo \"`echo \\\"$1\\\"`\"" +prop_checkSpacefulnessCfg32= verifyNot checkSpacefulnessCfg "var=$1; [ -v var ]" +prop_checkSpacefulnessCfg33= verify checkSpacefulnessCfg "for file; do echo $file; done" +prop_checkSpacefulnessCfg34= verify checkSpacefulnessCfg "declare foo$n=$1" +prop_checkSpacefulnessCfg35= verifyNot checkSpacefulnessCfg "echo ${1+\"$1\"}" +prop_checkSpacefulnessCfg36= verifyNot checkSpacefulnessCfg "arg=$#; echo $arg" +prop_checkSpacefulnessCfg37= verifyNot checkSpacefulnessCfg "@test 'status' {\n [ $status -eq 0 ]\n}" +prop_checkSpacefulnessCfg37v = verify checkVerboseSpacefulnessCfg "@test 'status' {\n [ $status -eq 0 ]\n}" +prop_checkSpacefulnessCfg38= verify checkSpacefulnessCfg "a=; echo $a" +prop_checkSpacefulnessCfg39= verifyNot checkSpacefulnessCfg "a=''\"\"''; b=x$a; echo $b" +prop_checkSpacefulnessCfg40= verifyNot checkSpacefulnessCfg "a=$((x+1)); echo $a" +prop_checkSpacefulnessCfg41= verifyNot checkSpacefulnessCfg "exec $1 --flags" +prop_checkSpacefulnessCfg42= verifyNot checkSpacefulnessCfg "run $1 --flags" +prop_checkSpacefulnessCfg43= verifyNot checkSpacefulnessCfg "$foo=42" +prop_checkSpacefulnessCfg44= verify checkSpacefulnessCfg "#!/bin/sh\nexport var=$value" +prop_checkSpacefulnessCfg45= verifyNot checkSpacefulnessCfg "wait -zzx -p foo; echo $foo" +prop_checkSpacefulnessCfg46= verifyNot checkSpacefulnessCfg "x=0; (( x += 1 )); echo $x" +prop_checkSpacefulnessCfg47= verifyNot checkSpacefulnessCfg "x=0; (( x-- )); echo $x" +prop_checkSpacefulnessCfg48= verifyNot checkSpacefulnessCfg "x=0; (( ++x )); echo $x" +prop_checkSpacefulnessCfg49= verifyNot checkSpacefulnessCfg "for i in 1 2 3; do echo $i; done" +prop_checkSpacefulnessCfg50= verify checkSpacefulnessCfg "for i in 1 2 *; do echo $i; done" +prop_checkSpacefulnessCfg51= verify checkSpacefulnessCfg "x='foo bar'; x && x=1; echo $x" +prop_checkSpacefulnessCfg52= verifyNot checkSpacefulnessCfg "x=1; if f; then x='foo bar'; exit; fi; echo $x" +prop_checkSpacefulnessCfg53= verifyNot checkSpacefulnessCfg "s=1; f() { local s='a b'; }; f; echo $s" +prop_checkSpacefulnessCfg54= verifyNot checkSpacefulnessCfg "s='a b'; f() { s=1; }; f; echo $s" +prop_checkSpacefulnessCfg55= verify checkSpacefulnessCfg "s='a b'; x && f() { s=1; }; f; echo $s" +prop_checkSpacefulnessCfg56= verifyNot checkSpacefulnessCfg "s=1; cat <(s='a b'); echo $s" + +checkSpacefulnessCfg = checkSpacefulnessCfg' True +checkVerboseSpacefulnessCfg = checkSpacefulnessCfg' False + +checkSpacefulnessCfg' :: Bool -> (Parameters -> Token -> Writer [TokenComment] ()) +checkSpacefulnessCfg' dirtyPass params token@(T_DollarBraced id _ list) = + when (needsQuoting && (dirtyPass == not isClean)) $ + unless (name `elem` specialVariablesWithoutSpaces || quotesMayConflictWithSC2281 params token) $ + if dirtyPass + then + if isDefaultAssignment (parentMap params) token + then + info (getId token) 2223 + "This default assignment may cause DoS due to globbing. Quote it." + else + infoWithFix id 2086 "Double quote to prevent globbing and word splitting." $ + addDoubleQuotesAround params token + else + styleWithFix id 2248 "Prefer double quoting even when variables don't contain special characters." $ + addDoubleQuotesAround params token + where - defaults = zip variablesWithoutSpaces (repeat SpaceNone) - - hasSpaces name = gets (Map.findWithDefault SpaceSome name) - - setSpaces name status = - modify $ Map.insert name status - - readF _ token name = do - spaces <- hasSpaces name - let needsQuoting = - isExpansion token - && not (isArrayExpansion token) -- There's another warning for this - && not (isCountingReference token) - && not (isQuoteFree (shellType params) parents token) - && not (isQuotedAlternativeReference token) - && not (usedAsCommandName parents token) - - return . execWriter $ when needsQuoting $ onFind spaces token name - - where - emit x = tell [x] - - writeF _ _ name (DataString SourceExternal) = setSpaces name SpaceSome >> return [] - writeF _ _ name (DataString SourceInteger) = setSpaces name SpaceNone >> return [] - - writeF _ _ name (DataString (SourceFrom vals)) = do - map <- get - setSpaces name - (isSpacefulWord (\x -> Map.findWithDefault SpaceSome x map) vals) - return [] - - writeF _ _ _ _ = return [] - + name = getBracedReference $ concat $ oversimplify list parents = parentMap params + needsQuoting = + not (isArrayExpansion token) -- There's another warning for this + && not (isCountingReference token) + && not (isQuoteFree (shellType params) parents token) + && not (isQuotedAlternativeReference token) + && not (usedAsCommandName parents token) - isExpansion t = - case t of - (T_DollarBraced _ _ _ ) -> True - _ -> False + isClean = fromMaybe False $ do + state <- CF.getIncomingState (cfgAnalysis params) id + value <- Map.lookup name $ CF.variablesInScope state + return $ CF.spaceStatus value == CF.SpaceStatusClean + + isDefaultAssignment parents token = + let modifier = getBracedModifier $ bracedString token in + any (`isPrefixOf` modifier) ["=", ":="] + && isParamTo parents ":" token + + -- Given a T_DollarBraced, return a simplified version of the string contents. + bracedString (T_DollarBraced _ _ l) = concat $ oversimplify l + bracedString _ = error $ pleaseReport "bracedString on non-variable" + +checkSpacefulnessCfg' _ _ _ = return () - isSpacefulWord :: (String -> SpaceStatus) -> [Token] -> SpaceStatus - isSpacefulWord f = mconcat . map (isSpaceful f) - isSpaceful :: (String -> SpaceStatus) -> Token -> SpaceStatus - isSpaceful spacefulF x = - case x of - T_DollarExpansion _ _ -> SpaceSome - T_Backticked _ _ -> SpaceSome - T_Glob _ _ -> SpaceSome - T_Extglob {} -> SpaceSome - T_DollarArithmetic _ _ -> SpaceNone - T_Literal _ s -> fromLiteral s - T_SingleQuoted _ s -> fromLiteral s - T_DollarBraced _ _ l -> spacefulF $ getBracedReference $ concat $ oversimplify l - T_NormalWord _ w -> isSpacefulWord spacefulF w - T_DoubleQuoted _ w -> isSpacefulWord spacefulF w - _ -> SpaceEmpty - where - globspace = "*?[] \t\n" - containsAny s = any (`elem` s) - fromLiteral "" = SpaceEmpty - fromLiteral s | s `containsAny` globspace = SpaceSome - fromLiteral _ = SpaceNone prop_CheckVariableBraces1 = verify checkVariableBraces "a='123'; echo $a" prop_CheckVariableBraces2 = verifyNot checkVariableBraces "a='123'; echo ${a}" From 8dc0fdb4cc3ae7cdea693b5dd2bead54fad7422e Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 20 Jul 2022 12:43:28 -0700 Subject: [PATCH 545/763] Precompile new fgl dependency on armv6hf --- build/linux.armv6hf/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/build/linux.armv6hf/Dockerfile b/build/linux.armv6hf/Dockerfile index bd5795c..f933dda 100644 --- a/build/linux.armv6hf/Dockerfile +++ b/build/linux.armv6hf/Dockerfile @@ -52,6 +52,7 @@ RUN pirun apt-get install -y ghc cabal-install ENV CABALOPTS "--ghc-options;-split-sections -optc-Os -optc-Wl,--gc-sections;--gcc-options;-Os -Wl,--gc-sections -ffunction-sections -fdata-sections" RUN pirun cabal update RUN IFS=";" && pirun cabal install --dependencies-only $CABALOPTS ShellCheck +RUN IFS=';' && pirun cabal install $CABALOPTS --lib fgl # Copy the build script WORKDIR /pi/scratch From 3ee4419ef4f4cda211401b1e97898dda5eb2e684 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 21 Jul 2022 15:06:05 -0700 Subject: [PATCH 546/763] Suppress SC2086 for variables declared -i (ref #2541) --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 16 +- src/ShellCheck/CFG.hs | 102 ++++++---- src/ShellCheck/CFGAnalysis.hs | 365 ++++++++++++++++++++++++++-------- 4 files changed, 358 insertions(+), 126 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 66d5369..7da6077 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Fixed - SC2086: Now uses DFA to make more accurate predictions about values +- SC2086: No longer warns about values declared as integer with declare -i ### Changed - ShellCheck now has a Data Flow Analysis engine to make smarter decisions diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 0062879..ca918ba 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -24,6 +24,7 @@ module ShellCheck.Analytics (runAnalytics, optionalChecks, ShellCheck.Analytics. import ShellCheck.AST import ShellCheck.ASTLib import ShellCheck.AnalyzerLib hiding (producesComments) +import ShellCheck.CFG import qualified ShellCheck.CFGAnalysis as CF import ShellCheck.Data import ShellCheck.Parser @@ -46,6 +47,7 @@ import Data.Ord import Data.Semigroup import Debug.Trace -- STRIP import qualified Data.Map.Strict as Map +import qualified Data.Set as S import Test.QuickCheck.All (forAllProperties) import Test.QuickCheck.Test (quickCheckWithResult, stdArgs, maxSuccess) @@ -2076,6 +2078,14 @@ prop_checkSpacefulnessCfg53= verifyNot checkSpacefulnessCfg "s=1; f() { local s= prop_checkSpacefulnessCfg54= verifyNot checkSpacefulnessCfg "s='a b'; f() { s=1; }; f; echo $s" prop_checkSpacefulnessCfg55= verify checkSpacefulnessCfg "s='a b'; x && f() { s=1; }; f; echo $s" prop_checkSpacefulnessCfg56= verifyNot checkSpacefulnessCfg "s=1; cat <(s='a b'); echo $s" +prop_checkSpacefulnessCfg57= verifyNot checkSpacefulnessCfg "declare -i s=0; s=$(f); echo $s" +prop_checkSpacefulnessCfg58= verify checkSpacefulnessCfg "f() { declare -i s; }; f; s=$(var); echo $s" +prop_checkSpacefulnessCfg59= verifyNot checkSpacefulnessCfg "f() { declare -gi s; }; f; s=$(var); echo $s" +prop_checkSpacefulnessCfg60= verify checkSpacefulnessCfg "declare -i s; declare +i s; s=$(foo); echo $s" +prop_checkSpacefulnessCfg61= verify checkSpacefulnessCfg "declare -x X; y=foo$X; echo $y;" +prop_checkSpacefulnessCfg62= verifyNot checkSpacefulnessCfg "f() { declare -x X; y=foo$X; echo $y; }" +prop_checkSpacefulnessCfg63= verify checkSpacefulnessCfg "f && declare -i s; s='x + y'; echo $s" +prop_checkSpacefulnessCfg64= verifyNot checkSpacefulnessCfg "declare -i s; s='x + y'; x=$s; echo $x" checkSpacefulnessCfg = checkSpacefulnessCfg' True checkVerboseSpacefulnessCfg = checkSpacefulnessCfg' False @@ -2110,7 +2120,11 @@ checkSpacefulnessCfg' dirtyPass params token@(T_DollarBraced id _ list) = isClean = fromMaybe False $ do state <- CF.getIncomingState (cfgAnalysis params) id value <- Map.lookup name $ CF.variablesInScope state - return $ CF.spaceStatus value == CF.SpaceStatusClean + return $ isCleanState value + + isCleanState state = + (all (S.member CFVPInteger) $ CF.variableProperties state) + || CF.spaceStatus (CF.variableValue state) == CF.SpaceStatusClean isDefaultAssignment parents token = let modifier = getBracedModifier $ bracedString token in diff --git a/src/ShellCheck/CFG.hs b/src/ShellCheck/CFG.hs index 101a0d7..a4bd166 100644 --- a/src/ShellCheck/CFG.hs +++ b/src/ShellCheck/CFG.hs @@ -32,6 +32,7 @@ module ShellCheck.CFG ( CFGraph, CFGParameters (..), IdTagged (..), + Scope (..), buildGraph , ShellCheck.CFG.runTests -- STRIP ) @@ -105,7 +106,8 @@ data CFEdge = -- Actions we track data CFEffect = - CFModifyProps String [CFVariableProp] + CFSetProps Scope String (S.Set CFVariableProp) + | CFUnsetProps Scope String (S.Set CFVariableProp) | CFReadVariable String | CFWriteVariable String CFValue | CFWriteGlobal String CFValue @@ -143,7 +145,7 @@ data CFValue = data CFStringPart = -- A known literal string value, like 'foo' CFStringLiteral String - -- The contents of a variable, like $foo + -- The contents of a variable, like $foo (may not be a string) | CFStringVariable String -- An value that is unknown but an integer | CFStringInteger @@ -152,7 +154,7 @@ data CFStringPart = deriving (Eq, Ord, Show, Generic, NFData) -- The properties of a variable -data CFVariableProp = CFVPExport | CFVPArray +data CFVariableProp = CFVPExport | CFVPArray | CFVPAssociative | CFVPInteger deriving (Eq, Ord, Show, Generic, NFData) -- Options when generating CFG @@ -961,71 +963,92 @@ handleCommand cmd vars args literalCmd = do handleDeclare (cmd:args) = do isFunc <- asks cfIsFunction - let (evaluated, effects) = mconcat $ map (toEffects isFunc) args + -- This is a bit of a kludge: we don't have great support for things like + -- 'declare -i x=$x' so do one round with declare x=$x, followed by declare -i x + let (evaluated, assignments, added, removed) = mconcat $ map (toEffects isFunc) args before <- sequentially $ evaluated - effect <- newNodeRange $ CFApplyEffects effects + assignments <- newNodeRange $ CFApplyEffects assignments + addedProps <- if null added then newStructuralNode else newNodeRange $ CFApplyEffects added + removedProps <- if null removed then newStructuralNode else newNodeRange $ CFApplyEffects removed result <- newNodeRange $ CFSetExitCode (getId cmd) - linkRanges [before, effect, result] + linkRanges [before, assignments, addedProps, removedProps, result] where opts = map fst $ getGenericOpts args - array = "a" `elem` opts || "A" `elem` opts + array = "a" `elem` opts || associative + associative = "A" `elem` opts integer = "i" `elem` opts func = "f" `elem` opts || "F" `elem` opts global = "g" `elem` opts + export = "x" `elem` opts writer isFunc = case () of _ | global -> CFWriteGlobal _ | isFunc -> CFWriteLocal _ -> CFWriteVariable - toEffects :: Bool -> Token -> ([Token], [IdTagged CFEffect]) + scope isFunc = + case () of + _ | global -> GlobalScope + _ | isFunc -> LocalScope + _ -> DefaultScope + + addedProps = S.fromList $ concat $ [ + [ CFVPArray | array ], + [ CFVPInteger | integer ], + [ CFVPExport | export ], + [ CFVPAssociative | associative ] + ] + + removedProps = S.fromList $ concat $ [ + -- Array property can't be unset + [ CFVPInteger | 'i' `elem` unsetOptions ], + [ CFVPExport | 'e' `elem` unsetOptions ] + ] + toEffects isFunc (T_Assignment id mode var idx t) = let pre = idx ++ [t] - isArray = array || (not $ null idx) - asArray = [ IdTagged id $ (writer isFunc) var CFValueArray ] - asString = [ IdTagged id $ (writer isFunc) var $ - if integer - then CFValueInteger -- TODO: Also handle integer variable property - else CFValueComputed (getId t) $ [ CFStringVariable var | mode == Append ] ++ tokenToParts t - ] + val = [ IdTagged id $ (writer isFunc) var $ CFValueComputed (getId t) $ [ CFStringVariable var | mode == Append ] ++ tokenToParts t ] + added = [ IdTagged id $ CFSetProps (scope isFunc) var addedProps | not $ S.null addedProps ] + removed = [ IdTagged id $ CFUnsetProps (scope isFunc) var addedProps | not $ S.null removedProps ] in - (pre, if isArray then asArray else asString ) + (pre, val, added, removed) toEffects isFunc t = let + id = getId t pre = [t] literal = fromJust $ getLiteralStringExt (const $ Just "\0") t isKnown = '\0' `notElem` literal match = fmap head $ variableAssignRegex `matchRegex` literal name = fromMaybe literal match - typer def = - if array - then CFValueArray - else - if integer - then CFValueInteger - else def + asLiteral = + IdTagged id $ (writer isFunc) name $ + CFValueComputed (getId t) [ CFStringLiteral $ drop 1 $ dropWhile (/= '=') $ literal ] + asUnknown = + IdTagged id $ (writer isFunc) name $ + CFValueString + + added = [ IdTagged id $ CFSetProps (scope isFunc) name addedProps ] + removed = [ IdTagged id $ CFUnsetProps (scope isFunc) name removedProps ] - asLiteral = [ - IdTagged (getId t) $ (writer isFunc) name $ - typer $ CFValueComputed (getId t) [ CFStringLiteral $ drop 1 $ dropWhile (/= '=') $ literal ] - ] - asUnknown = [ - IdTagged (getId t) $ (writer isFunc) name $ - typer $ CFValueString - ] - asBlank = [ - IdTagged (getId t) $ (writer isFunc) name $ - typer $ CFValueComputed (getId t) [] - ] in case () of - _ | not (isVariableName name) -> (pre, []) - _ | isJust match && isKnown -> (pre, asLiteral) - _ | isJust match -> (pre, asUnknown) - _ -> (pre, asBlank) + _ | not (isVariableName name) -> (pre, [], [], []) + _ | isJust match && isKnown -> (pre, [asLiteral], added, removed) + _ | isJust match -> (pre, [asUnknown], added, removed) + -- e.g. declare -i x + _ -> (pre, [], added, removed) + + -- find "ia" from `define +i +a` + unsetOptions :: String + unsetOptions = + let + strings = mapMaybe getLiteralString args + plusses = filter ("+" `isPrefixOf`) strings + in + concatMap (drop 1) plusses handlePrintf (cmd:args) = newNodeRange $ CFApplyEffects $ maybeToList findVar @@ -1103,6 +1126,7 @@ handleCommand cmd vars args literalCmd = do none = newStructuralNode data Scope = DefaultScope | GlobalScope | LocalScope | PrefixScope + deriving (Eq, Ord, Show, Generic, NFData) buildAssignment scope t = do op <- case t of diff --git a/src/ShellCheck/CFGAnalysis.hs b/src/ShellCheck/CFGAnalysis.hs index 99ce450..0007a67 100644 --- a/src/ShellCheck/CFGAnalysis.hs +++ b/src/ShellCheck/CFGAnalysis.hs @@ -50,7 +50,9 @@ module ShellCheck.CFGAnalysis ( ,CFGParameters (..) ,CFGAnalysis (..) ,ProgramState (..) + ,VariableState (..) ,VariableValue (..) + ,VariableProperties ,SpaceStatus (..) ,getIncomingState ,getOutgoingState @@ -77,9 +79,21 @@ import Debug.Trace -- STRIP import Test.QuickCheck +-- The number of iterations for DFA to stabilize iterationCount = 1000000 +-- There have been multiple bugs where bad caching caused oscillations. +-- As a precaution, disable caching if there's this many iterations left. +fallbackThreshold = 10000 +-- The number of cache entries to keep per node cacheEntries = 10 +logVerbose log = do + -- traceShowM log + return () +logInfo log = do + -- traceShowM log + return () + -- The result of the data flow analysis data CFGAnalysis = CFGAnalysis { graph :: CFGraph, @@ -89,9 +103,9 @@ data CFGAnalysis = CFGAnalysis { -- The program state we expose externally data ProgramState = ProgramState { - variablesInScope :: M.Map String VariableValue, +-- internalState :: InternalState, -- For debugging + variablesInScope :: M.Map String VariableState, stateIsReachable :: Bool --- internalState :: InternalState } deriving (Show, Eq, Generic, NFData) -- Conveniently get the state before a token id @@ -111,9 +125,9 @@ getDataForNode analysis node = M.lookup node $ nodeToData analysis -- The current state of data flow at a point in the program, potentially as a diff data InternalState = InternalState { sVersion :: Integer, - sGlobalValues :: VersionedMap String VariableValue, - sLocalValues :: VersionedMap String VariableValue, - sPrefixValues :: VersionedMap String VariableValue, + sGlobalValues :: VersionedMap String VariableState, + sLocalValues :: VersionedMap String VariableState, + sPrefixValues :: VersionedMap String VariableState, sFunctionTargets :: VersionedMap String FunctionValue, sIsReachable :: Maybe Bool } deriving (Show, Generic, NFData) @@ -135,31 +149,33 @@ unreachableState = modified newInternalState { createEnvironmentState :: InternalState createEnvironmentState = do foldl' (flip ($)) newInternalState $ concat [ - addVars Data.internalVariables unknownVariableValue, - addVars Data.variablesWithoutSpaces spacelessVariableValue, - addVars Data.specialIntegerVariables spacelessVariableValue + addVars Data.internalVariables unknownVariableState, + addVars Data.variablesWithoutSpaces spacelessVariableState, + addVars Data.specialIntegerVariables spacelessVariableState ] where addVars names val = map (\name -> insertGlobal name val) names - spacelessVariableValue = VariableValue { - literalValue = Nothing, - spaceStatus = SpaceStatusClean + spacelessVariableState = unknownVariableState { + variableValue = VariableValue { + literalValue = Nothing, + spaceStatus = SpaceStatusClean + } } modified s = s { sVersion = -1 } -insertGlobal :: String -> VariableValue -> InternalState -> InternalState +insertGlobal :: String -> VariableState -> InternalState -> InternalState insertGlobal name value state = modified state { sGlobalValues = vmInsert name value $ sGlobalValues state } -insertLocal :: String -> VariableValue -> InternalState -> InternalState +insertLocal :: String -> VariableState -> InternalState -> InternalState insertLocal name value state = modified state { sLocalValues = vmInsert name value $ sLocalValues state } -insertPrefix :: String -> VariableValue -> InternalState -> InternalState +insertPrefix :: String -> VariableState -> InternalState -> InternalState insertPrefix name value state = modified state { sPrefixValues = vmInsert name value $ sPrefixValues state } @@ -169,24 +185,38 @@ insertFunction name value state = modified state { sFunctionTargets = vmInsert name value $ sFunctionTargets state } +addProperties :: S.Set CFVariableProp -> VariableState -> VariableState +addProperties props state = state { + variableProperties = S.map (S.union props) $ variableProperties state +} + +removeProperties :: S.Set CFVariableProp -> VariableState -> VariableState +removeProperties props state = state { + variableProperties = S.map (\s -> S.difference s props) $ variableProperties state +} + internalToExternal :: InternalState -> ProgramState internalToExternal s = ProgramState { - -- Avoid introducing dependencies on the literal value as this is only for debugging purposes right now - variablesInScope = M.map (\c -> c { literalValue = Nothing }) flatVars, + -- Censor the literal value to avoid introducing dependencies on it. It's just for debugging. + variablesInScope = M.map censor flatVars, -- internalState = s, -- For debugging stateIsReachable = fromMaybe True $ sIsReachable s } where + censor s = s { + variableValue = (variableValue s) { + literalValue = Nothing + } + } flatVars = M.unionsWith (\_ last -> last) $ map mapStorage [sGlobalValues s, sLocalValues s, sPrefixValues s] -- Dependencies on values, e.g. "if there is a global variable named 'foo' without spaces" -- This is used to see if the DFA of a function would result in the same state, so anything -- that affects DFA must be tracked. data StateDependency = - DepGlobalValue String VariableValue - | DepLocalValue String VariableValue - | DepPrefixValue String VariableValue + DepState Scope String VariableState + | DepProperties Scope String VariableProperties | DepFunction String (S.Set FunctionDefinition) -- Whether invoking the node would result in recursion (i.e., is the function on the stack?) | DepIsRecursive Node Bool @@ -199,10 +229,6 @@ data FunctionDefinition = FunctionUnknown | FunctionDefinition String Node Node -- The Set of places a command name can point (it's a Set to handle conditionally defined functions) type FunctionValue = S.Set FunctionDefinition --- The scope of a function. ("Prefix" refers to e.g. `foo=1 env`) -data VariableScope = PrefixVar | LocalVar | GlobalVar - deriving (Show, Eq, Ord, Generic, NFData) - -- Create an InternalState that fulfills the given dependencies depsToState :: S.Set StateDependency -> InternalState depsToState set = foldl insert newInternalState $ S.toList set @@ -211,11 +237,26 @@ depsToState set = foldl insert newInternalState $ S.toList set insert state dep = case dep of DepFunction name val -> insertFunction name val state - DepGlobalValue name val -> insertGlobal name val state - DepLocalValue name val -> insertLocal name val state - DepPrefixValue name val -> insertPrefix name val state + DepState scope name val -> insertIn True scope name val state + -- State includes properties and more, so don't overwrite a state with properties + DepProperties scope name props -> insertIn False scope name unknownVariableState { variableProperties = props } state DepIsRecursive _ _ -> state + insertIn overwrite scope name val state = + let + (mapToCheck, inserter) = + case scope of + PrefixScope -> (sPrefixValues, insertPrefix) + LocalScope -> (sLocalValues, insertLocal) + GlobalScope -> (sGlobalValues, insertGlobal) + DefaultScope -> error $ pleaseReport "Unresolved scope in dependency" + + alreadyExists = isJust $ vmLookup name $ mapToCheck state + in + if overwrite || not alreadyExists + then inserter name val state + else state + unknownFunctionValue = S.singleton FunctionUnknown -- The information about the value of a single variable @@ -225,20 +266,45 @@ data VariableValue = VariableValue { } deriving (Show, Eq, Ord, Generic, NFData) +data VariableState = VariableState { + variableValue :: VariableValue, + variableProperties :: VariableProperties +} + deriving (Show, Eq, Ord, Generic, NFData) + -- Whether or not the value needs quoting (has spaces/globs), or we don't know data SpaceStatus = SpaceStatusEmpty | SpaceStatusClean | SpaceStatusDirty deriving (Show, Eq, Ord, Generic, NFData) +-- The set of possible sets of properties for this variable +type VariableProperties = S.Set (S.Set CFVariableProp) + +defaultProperties = S.singleton S.empty + +unknownVariableState = VariableState { + variableValue = unknownVariableValue, + variableProperties = defaultProperties +} unknownVariableValue = VariableValue { literalValue = Nothing, spaceStatus = SpaceStatusDirty } -emptyVariableValue = VariableValue { +emptyVariableValue = unknownVariableValue { literalValue = Just "", spaceStatus = SpaceStatusEmpty } +unsetVariableState = VariableState { + variableValue = emptyVariableValue, + variableProperties = defaultProperties +} + +mergeVariableState a b = VariableState { + variableValue = mergeVariableValue (variableValue a) (variableValue b), + variableProperties = S.union (variableProperties a) (variableProperties b) +} + mergeVariableValue a b = VariableValue { literalValue = if literalValue a == literalValue b then literalValue a else Nothing, spaceStatus = mergeSpaceStatus (spaceStatus a) (spaceStatus b) @@ -296,6 +362,8 @@ data Ctx s = Ctx { cCounter :: STRef s Integer, -- A cache of input state dependencies to output effects cCache :: STRef s (M.Map Node [(S.Set StateDependency, InternalState)]), + -- Whether the cache is enabled (see fallbackThreshold) + cEnableCache :: STRef s Bool, -- The states resulting from data flows per invocation path cInvocations :: STRef s (M.Map [Node] (S.Set StateDependency, M.Map Node (InternalState, InternalState))) } @@ -304,6 +372,8 @@ data Ctx s = Ctx { data StackEntry s = StackEntry { -- The entry point of this stack entry for the purpose of detecting recursion entryPoint :: Node, + -- Whether this is a function call (as opposed to a subshell) + isFunctionCall :: Bool, -- The node where this entry point was invoked callSite :: Node, -- A mutable set of dependencies we fetched from here or higher in the stack @@ -369,9 +439,9 @@ mergeState ctx a b = do return unreachableState _ | sVersion a >= 0 && sVersion b >= 0 && sVersion a == sVersion b -> return a _ -> do - globals <- mergeMaps ctx mergeVariableValue readGlobal (sGlobalValues a) (sGlobalValues b) - locals <- mergeMaps ctx mergeVariableValue readVariable (sLocalValues a) (sLocalValues b) - prefix <- mergeMaps ctx mergeVariableValue readVariable (sPrefixValues a) (sPrefixValues b) + globals <- mergeMaps ctx mergeVariableState readGlobal (sGlobalValues a) (sGlobalValues b) + locals <- mergeMaps ctx mergeVariableState readVariable (sLocalValues a) (sLocalValues b) + prefix <- mergeMaps ctx mergeVariableState readVariable (sPrefixValues a) (sPrefixValues b) funcs <- mergeMaps ctx S.union readFunction (sFunctionTargets a) (sFunctionTargets b) return $ InternalState { sVersion = -1, @@ -517,15 +587,15 @@ vmPatch base diff = mapStorage = M.unionWith (flip const) (mapStorage base) (mapStorage diff) } --- Modify a variable as with x=1. This applies it to the appropriate scope. -writeVariable :: forall s. Ctx s -> String -> VariableValue -> ST s () +-- Set a variable. This includes properties. Applies it to the appropriate scope. +writeVariable :: forall s. Ctx s -> String -> VariableState -> ST s () writeVariable ctx name val = do - (_, typ) <- readVariableWithScope ctx name + typ <- readVariableScope ctx name case typ of - GlobalVar -> writeGlobal ctx name val - LocalVar -> writeLocal ctx name val + GlobalScope -> writeGlobal ctx name val + LocalScope -> writeLocal ctx name val -- Prefixed variables actually become local variables in the invoked function - PrefixVar -> writeLocal ctx name val + PrefixScope -> writeLocal ctx name val writeGlobal ctx name val = do modifySTRef (cOutput ctx) $ insertGlobal name val @@ -536,39 +606,97 @@ writeLocal ctx name val = do writePrefix ctx name val = do modifySTRef (cOutput ctx) $ insertPrefix name val +updateVariableValue ctx name val = do + (props, scope) <- readVariablePropertiesWithScope ctx name + let f = case scope of + GlobalScope -> writeGlobal + LocalScope -> writeLocal + PrefixScope -> writeLocal -- Updates become local + f ctx name $ VariableState { variableValue = val, variableProperties = props } + +updateGlobalValue ctx name val = do + props <- readGlobalProperties ctx name + writeGlobal ctx name VariableState { variableValue = val, variableProperties = props } + +updateLocalValue ctx name val = do + props <- readLocalProperties ctx name + writeLocal ctx name VariableState { variableValue = val, variableProperties = props } + +updatePrefixValue ctx name val = do + -- Prefix variables don't inherit properties + writePrefix ctx name VariableState { variableValue = val, variableProperties = defaultProperties } + + -- Look up a variable value, and also return its scope -readVariableWithScope :: forall s. Ctx s -> String -> ST s (VariableValue, VariableScope) +readVariableWithScope :: forall s. Ctx s -> String -> ST s (VariableState, Scope) readVariableWithScope ctx name = lookupStack get dep def ctx name where - def = (unknownVariableValue, GlobalVar) + def = (unknownVariableState, GlobalScope) get = getVariableWithScope - dep k v = - case v of - (val, GlobalVar) -> DepGlobalValue k val - (val, LocalVar) -> DepLocalValue k val - (val, PrefixVar) -> DepPrefixValue k val + dep k (val, scope) = DepState scope k val -getVariableWithScope :: InternalState -> String -> Maybe (VariableValue, VariableScope) +-- Look up the variable's properties. This can be done independently to avoid incurring a dependency on the value. +readVariablePropertiesWithScope :: forall s. Ctx s -> String -> ST s (VariableProperties, Scope) +readVariablePropertiesWithScope ctx name = lookupStack get dep def ctx name + where + def = (defaultProperties, GlobalScope) + get s k = do + (val, scope) <- getVariableWithScope s k + return (variableProperties val, scope) + dep k (val, scope) = DepProperties scope k val + +readVariableScope ctx name = snd <$> readVariablePropertiesWithScope ctx name + +getVariableWithScope :: InternalState -> String -> Maybe (VariableState, Scope) getVariableWithScope s name = case (vmLookup name $ sPrefixValues s, vmLookup name $ sLocalValues s, vmLookup name $ sGlobalValues s) of - (Just var, _, _) -> return (var, PrefixVar) - (_, Just var, _) -> return (var, LocalVar) - (_, _, Just var) -> return (var, GlobalVar) + (Just var, _, _) -> return (var, PrefixScope) + (_, Just var, _) -> return (var, LocalScope) + (_, _, Just var) -> return (var, GlobalScope) _ -> Nothing undefineFunction ctx name = writeFunction ctx name $ FunctionUnknown undefineVariable ctx name = - writeVariable ctx name $ emptyVariableValue + writeVariable ctx name $ unsetVariableState readVariable ctx name = fst <$> readVariableWithScope ctx name +readVariableProperties ctx name = fst <$> readVariablePropertiesWithScope ctx name readGlobal ctx name = lookupStack get dep def ctx name where - def = unknownVariableValue + def = unknownVariableState -- could come from the environment get s name = vmLookup name $ sGlobalValues s - dep k v = DepGlobalValue k v + dep k v = DepState GlobalScope k v + + +readGlobalProperties ctx name = lookupStack get dep def ctx name + where + def = defaultProperties + get s name = variableProperties <$> (vmLookup name $ sGlobalValues s) + -- This dependency will fail to match if it's shadowed by a local variable, + -- such as in x=1; f() { local -i x; declare -ag x; } because we'll look at + -- x and find it to be local and not global. FIXME? + dep k v = DepProperties GlobalScope k v + +readLocal ctx name = lookupStackUntilFunction get dep def ctx name + where + def = unsetVariableState -- can't come from the environment + get s name = vmLookup name $ sLocalValues s + dep k v = DepState LocalScope k v + +-- We only want to look up the local properties of the current function, +-- though preferably even if we're in a subshell. FIXME? +readLocalProperties ctx name = fst <$> lookupStackUntilFunction get dep def ctx name + where + def = (defaultProperties, LocalScope) + with tag f = do + val <- variableProperties <$> f + return (val, tag) + + get s name = (with LocalScope $ vmLookup name $ sLocalValues s) `mplus` (with PrefixScope $ vmLookup name $ sPrefixValues s) + dep k (val, scope) = DepProperties scope k val readFunction ctx name = lookupStack get dep def ctx name where @@ -581,9 +709,11 @@ writeFunction ctx name val = do -- Look up each state on the stack until a value is found (or the default is used), -- then add this value as a StateDependency. -lookupStack :: forall s k v. +lookupStack' :: forall s k v. + -- Whether to stop at function boundaries + Bool -- A function that maybe finds a value from a state - (InternalState -> k -> Maybe v) + -> (InternalState -> k -> Maybe v) -- A function that creates a dependency on what was found -> (k -> v -> StateDependency) -- A default value, if the value can't be found anywhere @@ -594,13 +724,14 @@ lookupStack :: forall s k v. -> k -- Returning the result -> ST s v -lookupStack get dep def ctx key = do +lookupStack' functionOnly get dep def ctx key = do top <- readSTRef $ cInput ctx case get top key of Just v -> return v Nothing -> f (cStack ctx) where f [] = return def + f (s:_) | functionOnly && isFunctionCall s = return def f (s:rest) = do -- Go up the stack until we find the value, and add -- a dependency on each state (including where it was found) @@ -608,6 +739,9 @@ lookupStack get dep def ctx key = do modifySTRef (dependencies s) $ S.insert $ dep key res return res +lookupStack = lookupStack' False +lookupStackUntilFunction = lookupStack' True + -- Like lookupStack but without adding dependencies peekStack get def ctx key = do top <- readSTRef $ cInput ctx @@ -621,26 +755,30 @@ peekStack get def ctx key = do Just v -> return v Nothing -> f rest --- Check if the current context fulfills a StateDependency -fulfillsDependency ctx dep = +-- Check if the current context fulfills a StateDependency if entering `entry` +fulfillsDependency ctx entry dep = case dep of - DepGlobalValue name val -> (== (val, GlobalVar)) <$> peek ctx name - DepLocalValue name val -> (== (val, LocalVar)) <$> peek ctx name - DepPrefixValue name val -> (== (val, PrefixVar)) <$> peek ctx name + DepState scope name val -> (== (val, scope)) <$> peek scope ctx name + DepProperties scope name props -> do + (state, s) <- peek scope ctx name + return $ scope == s && variableProperties state == props DepFunction name val -> (== val) <$> peekFunc ctx name + -- Hack. Since we haven't pushed the soon-to-be invoked function on the stack, + -- it won't be found by the normal check. + DepIsRecursive node val | node == entry -> return True DepIsRecursive node val -> return $ val == any (\f -> entryPoint f == node) (cStack ctx) -- _ -> error $ "Unknown dep " ++ show dep where - peek = peekStack getVariableWithScope (unknownVariableValue, GlobalVar) + peek scope = peekStack getVariableWithScope $ if scope == GlobalScope then (unknownVariableState, GlobalScope) else (unsetVariableState, LocalScope) peekFunc = peekStack (\state name -> vmLookup name $ sFunctionTargets state) unknownFunctionValue -- Check if the current context fulfills all StateDependencies -fulfillsDependencies ctx deps = +fulfillsDependencies ctx entry deps = f $ S.toList deps where f [] = return True f (dep:rest) = do - res <- fulfillsDependency ctx dep + res <- fulfillsDependency ctx entry dep if res then f rest else return False @@ -652,6 +790,7 @@ newCtx g = do output <- newSTRef undefined node <- newSTRef undefined cache <- newSTRef M.empty + enableCache <- newSTRef True invocations <- newSTRef M.empty return $ Ctx { cCounter = c, @@ -659,6 +798,7 @@ newCtx g = do cOutput = output, cNode = node, cCache = cache, + cEnableCache = enableCache, cStack = [], cInvocations = invocations, cGraph = g @@ -672,20 +812,21 @@ nextVersion ctx = do return n -- Create a new StackEntry -newStackEntry ctx point = do +newStackEntry ctx point isCall = do deps <- newSTRef S.empty state <- readSTRef $ cOutput ctx callsite <- readSTRef $ cNode ctx return $ StackEntry { entryPoint = point, + isFunctionCall = isCall, callSite = callsite, dependencies = deps, stackState = state } -- Call a function with a new stack entry on the stack -withNewStackFrame ctx node f = do - newEntry <- newStackEntry ctx node +withNewStackFrame ctx node isCall f = do + newEntry <- newStackEntry ctx node isCall newInput <- newSTRef newInternalState newOutput <- newSTRef newInternalState newNode <- newSTRef node @@ -753,7 +894,7 @@ transferSubshell ctx reason entry exit = do writeSTRef cout initial where f entry exit ctx = do - (states, frame) <- withNewStackFrame ctx entry (flip dataflow $ entry) + (states, frame) <- withNewStackFrame ctx entry False (flip dataflow $ entry) let (_, res) = fromMaybe (error $ pleaseReport "Subshell has no exit") $ M.lookup exit states deps <- readSTRef $ dependencies frame registerFlowResult ctx entry states deps @@ -763,12 +904,12 @@ transferSubshell ctx reason entry exit = do transferCommand ctx Nothing = return () transferCommand ctx (Just name) = do targets <- readFunction ctx name - --traceShowM ("Transferring ",name,targets) + logVerbose ("Transferring ",name,targets) transferMultiple ctx $ map (flip transferFunctionValue) $ S.toList targets -- Transfer a set of function definitions and merge the output states. transferMultiple ctx funcs = do --- traceShowM ("Transferring set of ", length funcs) + logVerbose ("Transferring set of ", length funcs) original <- readSTRef out branches <- mapM (apply ctx original) funcs merged <- mergeStates ctx original branches @@ -792,7 +933,7 @@ transferFunctionValue ctx funcVal = else runCached ctx entry (f name entry exit) where f name entry exit ctx = do - (states, frame) <- withNewStackFrame ctx entry (flip dataflow $ entry) + (states, frame) <- withNewStackFrame ctx entry True (flip dataflow $ entry) deps <- readSTRef $ dependencies frame let res = case M.lookup exit states of @@ -827,25 +968,31 @@ runCached ctx node f = do cache <- getCache ctx node case cache of Just v -> do - -- traceShowM $ ("Running cached", node) + logInfo ("Running cached", node) + -- do { (deps, diff) <- f ctx; unless (v == diff) $ traceShowM ("Cache FAILED to match actual result", node, deps, diff); } patchOutputM ctx v + Nothing -> do - -- traceShowM $ ("Cache failed", node) + logInfo ("Cache failed", node) (deps, diff) <- f ctx modifySTRef (cCache ctx) (M.insertWith (\_ old -> (deps, diff):(take cacheEntries old)) node [(deps,diff)]) - -- traceShowM $ ("Recomputed cache for", node, deps) + logVerbose ("Recomputed cache for", node, deps) + -- do { f <- fulfillsDependencies ctx node deps; unless (f) $ traceShowM ("New dependencies FAILED to match", node, deps); } patchOutputM ctx diff -- Get a cached version whose dependencies are currently fulfilled, if any. getCache :: forall s. Ctx s -> Node -> ST s (Maybe InternalState) getCache ctx node = do cache <- readSTRef $ cCache ctx - -- traceShowM $ ("Cache for", node, "length", length $ M.findWithDefault [] node cache, M.lookup node cache) - f $ M.findWithDefault [] node cache + enable <- readSTRef $ cEnableCache ctx + logVerbose ("Cache for", node, "length", length $ M.findWithDefault [] node cache, M.lookup node cache) + if enable + then f $ M.findWithDefault [] node cache + else return Nothing where f [] = return Nothing f ((deps, value):rest) = do - match <- fulfillsDependencies ctx deps + match <- fulfillsDependencies ctx node deps if match then return $ Just value else f rest @@ -857,16 +1004,52 @@ transferEffect ctx effect = void $ readVariable ctx name CFWriteVariable name value -> do val <- cfValueToVariableValue ctx value - writeVariable ctx name val + updateVariableValue ctx name val CFWriteGlobal name value -> do val <- cfValueToVariableValue ctx value - writeGlobal ctx name val + updateGlobalValue ctx name val CFWriteLocal name value -> do val <- cfValueToVariableValue ctx value - writeLocal ctx name val + updateLocalValue ctx name val CFWritePrefix name value -> do val <- cfValueToVariableValue ctx value - writePrefix ctx name val + updatePrefixValue ctx name val + + CFSetProps scope name props -> + case scope of + DefaultScope -> do + state <- readVariable ctx name + writeVariable ctx name $ addProperties props state + GlobalScope -> do + state <- readGlobal ctx name + writeGlobal ctx name $ addProperties props state + LocalScope -> do + out <- readSTRef (cOutput ctx) + state <- readLocal ctx name + writeLocal ctx name $ addProperties props state + PrefixScope -> do + -- Prefix values become local + state <- readLocal ctx name + writeLocal ctx name $ addProperties props state + + CFUnsetProps scope name props -> + case scope of + DefaultScope -> do + state <- readVariable ctx name + writeVariable ctx name $ removeProperties props state + GlobalScope -> do + state <- readGlobal ctx name + writeGlobal ctx name $ removeProperties props state + LocalScope -> do + out <- readSTRef (cOutput ctx) + state <- readLocal ctx name + writeLocal ctx name $ removeProperties props state + PrefixScope -> do + -- Prefix values become local + state <- readLocal ctx name + writeLocal ctx name $ removeProperties props state + + CFUndefineVariable name -> undefineVariable ctx name CFUndefineFunction name -> undefineFunction ctx name CFUndefine name -> do @@ -880,11 +1063,9 @@ transferEffect ctx effect = CFUndefineNameref name -> undefineVariable ctx name CFHintArray name -> return () CFHintDefined name -> return () - CFModifyProps {} -> return () -- _ -> error $ "Unknown effect " ++ show effect - -- Transfer the CFG's idea of a value into our VariableState cfValueToVariableValue ctx val = case val of @@ -905,12 +1086,17 @@ computeValue ctx part = CFStringLiteral str -> return $ literalToVariableValue str CFStringInteger -> return unknownIntegerValue CFStringUnknown -> return unknownVariableValue - CFStringVariable name -> readVariable ctx name + CFStringVariable name -> variableStateToValue <$> readVariable ctx name + where + variableStateToValue state = + case () of + _ | all (CFVPInteger `S.member`) $ variableProperties state -> unknownIntegerValue + _ -> variableValue state -- Append two VariableValues as if with z="$x$y" appendVariableValue :: VariableValue -> VariableValue -> VariableValue appendVariableValue a b = - VariableValue { + unknownVariableValue { literalValue = liftM2 (++) (literalValue a) (literalValue b), spaceStatus = appendSpaceStatus (spaceStatus a) (spaceStatus b) } @@ -922,12 +1108,12 @@ appendSpaceStatus a b = (SpaceStatusClean, SpaceStatusClean) -> a _ ->SpaceStatusDirty -unknownIntegerValue = VariableValue { +unknownIntegerValue = unknownVariableValue { literalValue = Nothing, spaceStatus = SpaceStatusClean } -literalToVariableValue str = VariableValue { +literalToVariableValue str = unknownVariableValue { literalValue = Just str, spaceStatus = literalToSpaceStatus str } @@ -965,6 +1151,13 @@ dataflow ctx entry = do f 0 _ _ = error $ pleaseReport "DFA did not reach fix point" f n pending states = do ps <- readSTRef pending + + when (n == fallbackThreshold) $ do + -- This should never happen, but has historically been due to caching bugs. + -- Try disabling the cache and continuing. + logInfo "DFA is not stabilizing! Disabling cache." + writeSTRef (cEnableCache ctx) False + if S.null ps then return () else do @@ -1012,12 +1205,12 @@ runRoot ctx entry exit = do writeSTRef (cInput ctx) $ env writeSTRef (cOutput ctx) $ env writeSTRef (cNode ctx) $ entry - (states, frame) <- withNewStackFrame ctx entry $ \c -> dataflow c entry + (states, frame) <- withNewStackFrame ctx entry False $ \c -> dataflow c entry deps <- readSTRef $ dependencies frame registerFlowResult ctx entry states deps -- Return the final state, used to invoke functions that were declared but not invoked return $ snd $ fromMaybe (error $ pleaseReport "Missing exit state") $ M.lookup exit states - + analyzeControlFlow :: CFGParameters -> Token -> CFGAnalysis analyzeControlFlow params t = From e7f05d662ad1830cc9b4bce59a7b6767fa07d8c0 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Fri, 22 Jul 2022 10:29:19 -0700 Subject: [PATCH 547/763] In addition to start/end, track sets of nodes belonging to tokens --- src/ShellCheck/CFG.hs | 66 ++++++++++++++++++++++------------- src/ShellCheck/CFGAnalysis.hs | 12 ++++--- src/ShellCheck/Debug.hs | 2 +- 3 files changed, 50 insertions(+), 30 deletions(-) diff --git a/src/ShellCheck/CFG.hs b/src/ShellCheck/CFG.hs index a4bd166..4906d80 100644 --- a/src/ShellCheck/CFG.hs +++ b/src/ShellCheck/CFG.hs @@ -168,8 +168,10 @@ data CFGParameters = CFGParameters { data CFGResult = CFGResult { -- The graph itself cfGraph :: CFGraph, - -- Map from Id to start/end node - cfIdToNode :: M.Map Id (Node, Node) + -- Map from Id to nominal start&end node (i.e. assuming normal execution without exits) + cfIdToRange :: M.Map Id (Node, Node), + -- A set of all nodes belonging to an Id, recursively + cfIdToNodes :: M.Map Id (S.Set Node) } deriving (Show) @@ -177,21 +179,24 @@ buildGraph :: CFGParameters -> Token -> CFGResult buildGraph params root = let (nextNode, base) = execRWS (buildRoot root) (newCFContext params) 0 - (nodes, edges, mapping) = + (nodes, edges, mapping, association) = -- renumberTopologically $ removeUnnecessaryStructuralNodes base in CFGResult { cfGraph = mkGraph nodes edges, - cfIdToNode = M.fromList mapping + cfIdToRange = M.fromList mapping, + cfIdToNodes = M.fromListWith S.union $ map (\(id, n) -> (id, S.singleton n)) association } -remapGraph remap (nodes, edges, mapping) = +remapGraph :: M.Map Node Node -> CFW -> CFW +remapGraph remap (nodes, edges, mapping, assoc) = ( map (remapNode remap) nodes, map (remapEdge remap) edges, - map (\(id, (a,b)) -> (id, (remapHelper remap a, remapHelper remap b))) mapping + map (\(id, (a,b)) -> (id, (remapHelper remap a, remapHelper remap b))) mapping, + map (\(id, n) -> (id, remapHelper remap n)) assoc ) prop_testRenumbering = @@ -200,17 +205,20 @@ prop_testRenumbering = before = ( [(1,s), (3,s), (4, s), (8,s)], [(1,3,CFEFlow), (3,4, CFEFlow), (4,8,CFEFlow)], - [(Id 0, (3,4))] + [(Id 0, (3,4))], + [(Id 1, 3), (Id 2, 4)] ) after = ( [(0,s), (1,s), (2,s), (3,s)], [(0,1,CFEFlow), (1,2, CFEFlow), (2,3,CFEFlow)], - [(Id 0, (1,2))] + [(Id 0, (1,2))], + [(Id 1, 1), (Id 2, 2)] ) in after == renumberGraph before -- Renumber the graph for prettiness, so there are no gaps in node numbers -renumberGraph g@(nodes, edges, mapping) = +renumberGraph :: CFW -> CFW +renumberGraph g@(nodes, edges, mapping, assoc) = let renumbering = M.fromList (flip zip [0..] $ sort $ map fst nodes) in remapGraph renumbering g @@ -220,17 +228,19 @@ prop_testRenumberTopologically = before = ( [(4,s), (2,s), (3, s)], [(4,2,CFEFlow), (2,3, CFEFlow)], - [(Id 0, (4,2))] + [(Id 0, (4,2))], + [] ) after = ( [(0,s), (1,s), (2,s)], [(0,1,CFEFlow), (1,2, CFEFlow)], - [(Id 0, (0,1))] + [(Id 0, (0,1))], + [] ) in after == renumberTopologically before -- Renumber the graph in topological order -renumberTopologically g@(nodes, edges, mapping) = +renumberTopologically g@(nodes, edges, mapping, assoc) = let renumbering = M.fromList (flip zip [0..] $ topsort (mkGraph nodes edges :: CFGraph)) in remapGraph renumbering g @@ -240,12 +250,14 @@ prop_testRemoveStructural = before = ( [(1,s), (2,s), (3, s), (4,s)], [(1,2,CFEFlow), (2,3, CFEFlow), (3,4,CFEFlow)], - [(Id 0, (2,3))] + [(Id 0, (2,3))], + [(Id 0, 3)] ) after = ( [(1,s), (2,s), (4,s)], [(1,2,CFEFlow), (2,4,CFEFlow)], - [(Id 0, (2,2))] + [(Id 0, (2,2))], + [(Id 0, 2)] ) in after == removeUnnecessaryStructuralNodes before @@ -255,12 +267,13 @@ prop_testRemoveStructural = -- Note in particular that we can't remove a structural node x in -- foo -> x -> bar , because then the pre/post-condition for tokens -- previously pointing to x would be wrong. -removeUnnecessaryStructuralNodes (nodes, edges, mapping) = +removeUnnecessaryStructuralNodes (nodes, edges, mapping, association) = remapGraph recursiveRemapping ( filter (\(n, _) -> n `M.notMember` recursiveRemapping) nodes, filter (`S.notMember` edgesToCollapse) edges, - mapping + mapping, + association ) where regularEdges = filter isRegularEdge edges @@ -305,8 +318,6 @@ remapNode m (node, label) = newLabel = case label of CFApplyEffects effects -> CFApplyEffects (map (remapEffect m) effects) CFExecuteSubshell s a b -> CFExecuteSubshell s (remapHelper m a) (remapHelper m b) --- CFSubShellStart reason node -> CFSubShellStart reason (remapHelper m node) - _ -> label remapEffect map old@(IdTagged id effect) = @@ -325,6 +336,7 @@ data CFContext = CFContext { cfIsCondition :: Bool, cfIsFunction :: Bool, cfLoopStack :: [(Node, Node)], + cfTokenStack :: [Id], cfExitTarget :: Maybe Node, cfReturnTarget :: Maybe Node, cfParameters :: CFGParameters @@ -333,19 +345,22 @@ newCFContext params = CFContext { cfIsCondition = False, cfIsFunction = False, cfLoopStack = [], + cfTokenStack = [], cfExitTarget = Nothing, cfReturnTarget = Nothing, cfParameters = params } -- The monad we generate a graph in -type CFM a = RWS CFContext ([LNode CFNode], [LEdge CFEdge], [(Id, (Node, Node))]) Int a +type CFM a = RWS CFContext CFW Int a +type CFW = ([LNode CFNode], [LEdge CFEdge], [(Id, (Node, Node))], [(Id, Node)]) newNode :: CFNode -> CFM Node newNode label = do n <- get + stack <- asks cfTokenStack put (n+1) - tell ([(n, label)], [], []) + tell ([(n, label)], [], [], map (\c -> (c, n)) stack) return n newNodeRange :: CFNode -> CFM Range @@ -367,16 +382,19 @@ withFunctionScope p = do body <- local (\c -> c { cfReturnTarget = Just end, cfIsFunction = True }) p linkRanges [body, nodeToRange end] +-- Anything that happens recursively in f will be attributed to this id +under :: Id -> CFM a -> CFM a +under id f = local (\c -> c { cfTokenStack = id:(cfTokenStack c) }) f nodeToRange :: Node -> Range nodeToRange n = Range n n link :: Node -> Node -> CFEdge -> CFM () link from to label = do - tell ([], [(from, to, label)], []) + tell ([], [(from, to, label)], [], []) registerNode :: Id -> Range -> CFM () -registerNode id (Range start end) = tell ([], [], [(id, (start, end))]) +registerNode id (Range start end) = tell ([], [], [(id, (start, end))], []) linkRange :: Range -> Range -> CFM Range linkRange = linkRangeAs CFEFlow @@ -412,7 +430,7 @@ asCondition = withContext (\c -> c { cfIsCondition = True }) newStructuralNode = newNodeRange CFStructuralNode buildRoot :: Token -> CFM Range -buildRoot t = do +buildRoot t = under (getId t) $ do entry <- newNodeRange $ CFEntryPoint "MAIN" impliedExit <- newNode CFImpliedExit end <- newNode CFStructuralNode @@ -426,7 +444,7 @@ applySingle e = CFApplyEffects [e] -- Build the CFG. build :: Token -> CFM Range build t = do - range <- build' t + range <- under (getId t) $ build' t registerNode (getId t) range return range where diff --git a/src/ShellCheck/CFGAnalysis.hs b/src/ShellCheck/CFGAnalysis.hs index 0007a67..daade43 100644 --- a/src/ShellCheck/CFGAnalysis.hs +++ b/src/ShellCheck/CFGAnalysis.hs @@ -97,7 +97,8 @@ logInfo log = do -- The result of the data flow analysis data CFGAnalysis = CFGAnalysis { graph :: CFGraph, - tokenToNode :: M.Map Id (Node, Node), + tokenToRange :: M.Map Id (Node, Node), + tokenToNodes :: M.Map Id (S.Set Node), nodeToData :: M.Map Node (ProgramState, ProgramState) } deriving (Show, Generic, NFData) @@ -111,13 +112,13 @@ data ProgramState = ProgramState { -- Conveniently get the state before a token id getIncomingState :: CFGAnalysis -> Id -> Maybe ProgramState getIncomingState analysis id = do - (start,end) <- M.lookup id $ tokenToNode analysis + (start,end) <- M.lookup id $ tokenToRange analysis fst <$> M.lookup start (nodeToData analysis) -- Conveniently get the state after a token id getOutgoingState :: CFGAnalysis -> Id -> Maybe ProgramState getOutgoingState analysis id = do - (start,end) <- M.lookup id $ tokenToNode analysis + (start,end) <- M.lookup id $ tokenToRange analysis snd <$> M.lookup end (nodeToData analysis) getDataForNode analysis node = M.lookup node $ nodeToData analysis @@ -1216,7 +1217,7 @@ analyzeControlFlow :: CFGParameters -> Token -> CFGAnalysis analyzeControlFlow params t = let cfg = buildGraph params t - (entry, exit) = M.findWithDefault (error $ pleaseReport "Missing root") (getId t) (cfIdToNode cfg) + (entry, exit) = M.findWithDefault (error $ pleaseReport "Missing root") (getId t) (cfIdToRange cfg) in runST $ f cfg entry exit where @@ -1250,7 +1251,8 @@ analyzeControlFlow params t = return $ nodeToData `deepseq` CFGAnalysis { graph = cfGraph cfg, - tokenToNode = cfIdToNode cfg, + tokenToRange = cfIdToRange cfg, + tokenToNodes = cfIdToNodes cfg, nodeToData = nodeToData } diff --git a/src/ShellCheck/Debug.hs b/src/ShellCheck/Debug.hs index c991308..b6015e5 100644 --- a/src/ShellCheck/Debug.hs +++ b/src/ShellCheck/Debug.hs @@ -202,7 +202,7 @@ stringToDetailedCfgViz scriptString = cfgToGraphVizWith nodeLabel graph idToToken = M.fromList $ execWriter $ doAnalysis (\c -> tell [(getId c, c)]) ast idToNode :: M.Map Id (Node, Node) - idToNode = cfIdToNode cfgResult + idToNode = cfIdToRange cfgResult nodeToStartIds :: M.Map Node (S.Set Id) nodeToStartIds = From 95b3cbf0714df47feba9ff8d465c27cb6503f38f Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Fri, 22 Jul 2022 11:11:09 -0700 Subject: [PATCH 548/763] Qualify Data.Map as M instead of tedious Map --- src/ShellCheck/Checks/Commands.hs | 42 +++++++++++++++---------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index cac06bc..081ec5f 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -39,7 +39,7 @@ import Data.Char import Data.Functor.Identity import Data.List import Data.Maybe -import qualified Data.Map.Strict as Map +import qualified Data.Map.Strict as M import Test.QuickCheck.All (forAllProperties) import Test.QuickCheck.Test (quickCheckWithResult, stdArgs, maxSuccess) @@ -114,7 +114,7 @@ optionalCommandChecks = [ cdNegative = "command -v javac" }, checkWhich) ] -optionalCheckMap = Map.fromList $ map (\(desc, check) -> (cdName desc, check)) optionalCommandChecks +optionalCheckMap = M.fromList $ map (\(desc, check) -> (cdName desc, check)) optionalCommandChecks prop_verifyOptionalExamples = all check optionalCommandChecks where @@ -163,27 +163,27 @@ prop_checkGenericOptsT1 = checkGetOpts "-x -- -y" ["x"] ["-y"] $ return . getGen prop_checkGenericOptsT2 = checkGetOpts "-xy --" ["x", "y"] [] $ return . getGenericOpts -buildCommandMap :: [CommandCheck] -> Map.Map CommandName (Token -> Analysis) -buildCommandMap = foldl' addCheck Map.empty +buildCommandMap :: [CommandCheck] -> M.Map CommandName (Token -> Analysis) +buildCommandMap = foldl' addCheck M.empty where addCheck map (CommandCheck name function) = - Map.insertWith composeAnalyzers name function map + M.insertWith composeAnalyzers name function map -checkCommand :: Map.Map CommandName (Token -> Analysis) -> Token -> Analysis +checkCommand :: M.Map CommandName (Token -> Analysis) -> Token -> Analysis checkCommand map t@(T_SimpleCommand id cmdPrefix (cmd:rest)) = sequence_ $ do name <- getLiteralString cmd return $ if '/' `elem` name then - Map.findWithDefault nullCheck (Basename $ basename name) map t + M.findWithDefault nullCheck (Basename $ basename name) map t else if name == "builtin" && not (null rest) then let t' = T_SimpleCommand id cmdPrefix rest selectedBuiltin = fromMaybe "" $ getLiteralString . head $ rest - in Map.findWithDefault nullCheck (Exactly selectedBuiltin) map t' + in M.findWithDefault nullCheck (Exactly selectedBuiltin) map t' else do - Map.findWithDefault nullCheck (Exactly name) map t - Map.findWithDefault nullCheck (Basename name) map t + M.findWithDefault nullCheck (Exactly name) map t + M.findWithDefault nullCheck (Basename name) map t where basename = reverse . takeWhile (/= '/') . reverse @@ -205,7 +205,7 @@ checker spec params = getChecker $ commandChecks ++ optionals optionals = if "all" `elem` keys then map snd optionalCommandChecks - else mapMaybe (\x -> Map.lookup x optionalCheckMap) keys + else mapMaybe (\x -> M.lookup x optionalCheckMap) keys prop_checkTr1 = verify checkTr "tr [a-f] [A-F]" prop_checkTr2 = verify checkTr "tr 'a-z' 'A-Z'" @@ -1005,20 +1005,20 @@ checkWhileGetoptsCase = CommandCheck (Exactly "getopts") f check :: Id -> [String] -> Token -> Analysis check optId opts (T_CaseExpression id _ list) = do - unless (Nothing `Map.member` handledMap) $ do - mapM_ (warnUnhandled optId id) $ catMaybes $ Map.keys notHandled + unless (Nothing `M.member` handledMap) $ do + mapM_ (warnUnhandled optId id) $ catMaybes $ M.keys notHandled - unless (any (`Map.member` handledMap) [Just "*",Just "?"]) $ + unless (any (`M.member` handledMap) [Just "*",Just "?"]) $ warn id 2220 "Invalid flags are not handled. Add a *) case." - mapM_ warnRedundant $ Map.toList notRequested + mapM_ warnRedundant $ M.toList notRequested where - handledMap = Map.fromList (concatMap getHandledStrings list) - requestedMap = Map.fromList $ map (\x -> (Just x, ())) opts + handledMap = M.fromList (concatMap getHandledStrings list) + requestedMap = M.fromList $ map (\x -> (Just x, ())) opts - notHandled = Map.difference requestedMap handledMap - notRequested = Map.difference handledMap requestedMap + notHandled = M.difference requestedMap handledMap + notRequested = M.difference handledMap requestedMap warnUnhandled optId caseId str = warn caseId 2213 $ "getopts specified -" ++ (e4m str) ++ ", but it's not handled by this 'case'." @@ -1372,10 +1372,10 @@ checkUnquotedEchoSpaces = CommandCheck (Basename "echo") check m <- asks tokenPositions redir <- getClosestCommandM t sequence_ $ do - let positions = mapMaybe (\c -> Map.lookup (getId c) m) args + let positions = mapMaybe (\c -> M.lookup (getId c) m) args let pairs = zip positions (drop 1 positions) (T_Redirecting _ redirTokens _) <- redir - let redirPositions = mapMaybe (\c -> fst <$> Map.lookup (getId c) m) redirTokens + let redirPositions = mapMaybe (\c -> fst <$> M.lookup (getId c) m) redirTokens guard $ any (hasSpacesBetween redirPositions) pairs return $ info (getId t) 2291 "Quote repeated spaces to avoid them collapsing into one." From 9caeec104b5b70054c1b4238d4dfde4c641ea12e Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Fri, 22 Jul 2022 11:25:07 -0700 Subject: [PATCH 549/763] SC2318: Warn about backreferencing in `declare x=1 y=$x` (fixes #1653) --- CHANGELOG.md | 1 + src/ShellCheck/Checks/Commands.hs | 53 +++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7da6077..dce3e27 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ### Added - SC2316: Warn about 'local readonly foo' and similar (thanks, patrickxia!) - SC2317: Warn about unreachable commands +- SC2318: Warn about backreferences in 'declare x=1 y=$x' ### Fixed - SC2086: Now uses DFA to make more accurate predictions about values diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 081ec5f..acbf967 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -27,6 +27,8 @@ module ShellCheck.Checks.Commands (checker, optionalChecks, ShellCheck.Checks.Co import ShellCheck.AST import ShellCheck.ASTLib import ShellCheck.AnalyzerLib +import ShellCheck.CFG +import qualified ShellCheck.CFGAnalysis as CF import ShellCheck.Data import ShellCheck.Interface import ShellCheck.Parser @@ -37,12 +39,16 @@ import Control.Monad import Control.Monad.RWS import Data.Char import Data.Functor.Identity +import qualified Data.Graph.Inductive.Graph as G import Data.List import Data.Maybe import qualified Data.Map.Strict as M +import qualified Data.Set as S import Test.QuickCheck.All (forAllProperties) import Test.QuickCheck.Test (quickCheckWithResult, stdArgs, maxSuccess) +import Debug.Trace -- STRIP + data CommandName = Exactly String | Basename String deriving (Eq, Ord) @@ -102,6 +108,7 @@ commandChecks = [ ++ map checkArgComparison ("alias" : declaringCommands) ++ map checkMaskedReturns declaringCommands ++ map checkMultipleDeclaring declaringCommands + ++ map checkBackreferencingDeclaration declaringCommands optionalChecks = map fst optionalCommandChecks @@ -1405,5 +1412,51 @@ checkEvalArray = CommandCheck (Exactly "eval") (mapM_ check . concatMap getWordP _ -> False +prop_checkBackreferencingDeclaration1 = verify (checkBackreferencingDeclaration "declare") "declare x=1 y=foo$x" +prop_checkBackreferencingDeclaration2 = verify (checkBackreferencingDeclaration "readonly") "readonly x=1 y=$((1+x))" +prop_checkBackreferencingDeclaration3 = verify (checkBackreferencingDeclaration "local") "local x=1 y=$(echo $x)" +prop_checkBackreferencingDeclaration4 = verify (checkBackreferencingDeclaration "local") "local x=1 y[$x]=z" +prop_checkBackreferencingDeclaration5 = verify (checkBackreferencingDeclaration "declare") "declare x=var $x=1" +prop_checkBackreferencingDeclaration6 = verify (checkBackreferencingDeclaration "declare") "declare x=var $x=1" +prop_checkBackreferencingDeclaration7 = verify (checkBackreferencingDeclaration "declare") "declare x=var $k=$x" +checkBackreferencingDeclaration cmd = CommandCheck (Exactly cmd) check + where + check t = foldM_ perArg M.empty $ arguments t + + perArg leftArgs t = + case t of + T_Assignment id _ name idx t -> do + warnIfBackreferencing leftArgs $ t:idx + return $ M.insert name id leftArgs + t -> do + warnIfBackreferencing leftArgs [t] + return leftArgs + + warnIfBackreferencing backrefs l = do + references <- findReferences l + let reused = M.intersection backrefs references + mapM msg $ M.toList reused + + msg (name, id) = warn id 2318 $ "This assignment is used again in this '" ++ cmd ++ "', but won't have taken effect. Use two '" ++ cmd ++ "'s." + + findReferences list = do + cfga <- asks cfgAnalysis + let graph = CF.graph cfga + let nodesMap = CF.tokenToNodes cfga + let nodes = S.unions $ map (\id -> M.findWithDefault S.empty id nodesMap) $ map getId $ list + let labels = mapMaybe (G.lab graph) $ S.toList nodes + let references = M.fromList $ concatMap refFromLabel labels + return references + + refFromLabel lab = + case lab of + CFApplyEffects effects -> mapMaybe refFromEffect effects + _ -> [] + refFromEffect e = + case e of + IdTagged id (CFReadVariable name) -> return (name, id) + _ -> Nothing + + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) From e47480e93af07a7e72ee0eb8c0d7d57c92ec35e4 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Fri, 22 Jul 2022 16:28:24 -0700 Subject: [PATCH 550/763] Also emit SC2004 for array indices (fixes #1666) --- src/ShellCheck/Analytics.hs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index ca918ba..8b8d489 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1460,7 +1460,8 @@ prop_checkArithmeticDeref7 = verifyNot checkArithmeticDeref "(( 10#$n ))" prop_checkArithmeticDeref8 = verifyNot checkArithmeticDeref "let i=$i+1" prop_checkArithmeticDeref9 = verifyNot checkArithmeticDeref "(( a[foo] ))" prop_checkArithmeticDeref10= verifyNot checkArithmeticDeref "(( a[\\$foo] ))" -prop_checkArithmeticDeref11= verifyNot checkArithmeticDeref "a[$foo]=wee" +prop_checkArithmeticDeref11= verify checkArithmeticDeref "a[$foo]=wee" +prop_checkArithmeticDeref11b= verifyNot checkArithmeticDeref "declare -A a; a[$foo]=wee" prop_checkArithmeticDeref12= verify checkArithmeticDeref "for ((i=0; $i < 3; i)); do true; done" prop_checkArithmeticDeref13= verifyNot checkArithmeticDeref "(( $$ ))" prop_checkArithmeticDeref14= verifyNot checkArithmeticDeref "(( $! ))" @@ -1477,6 +1478,7 @@ checkArithmeticDeref params t@(TA_Expansion _ [T_DollarBraced id _ l]) = T_Arithmetic {} -> return normalWarning T_DollarArithmetic {} -> return normalWarning T_ForArithmetic {} -> return normalWarning + T_Assignment {} -> return normalWarning T_SimpleCommand {} -> return noWarning _ -> Nothing From 2f28847b0897a8f91c6f227ed098470eea383dd4 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Fri, 22 Jul 2022 16:35:14 -0700 Subject: [PATCH 551/763] Normalize spaces around = in unit tests --- src/ShellCheck/ASTLib.hs | 10 +- src/ShellCheck/Analytics.hs | 392 +++++++++++++------------- src/ShellCheck/Checks/Commands.hs | 76 ++--- src/ShellCheck/Checks/ShellSupport.hs | 106 +++---- src/ShellCheck/Parser.hs | 90 +++--- 5 files changed, 337 insertions(+), 337 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 7cc5af2..7b4f9e5 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -810,11 +810,11 @@ prop_getBracedReference6 = getBracedReference "!#" == "#" prop_getBracedReference7 = getBracedReference "!foo#?" == "foo" prop_getBracedReference8 = getBracedReference "foo-bar" == "foo" prop_getBracedReference9 = getBracedReference "foo:-bar" == "foo" -prop_getBracedReference10= getBracedReference "foo: -1" == "foo" -prop_getBracedReference11= getBracedReference "!os*" == "" -prop_getBracedReference11b= getBracedReference "!os@" == "" -prop_getBracedReference12= getBracedReference "!os?bar**" == "" -prop_getBracedReference13= getBracedReference "foo[bar]" == "foo" +prop_getBracedReference10 = getBracedReference "foo: -1" == "foo" +prop_getBracedReference11 = getBracedReference "!os*" == "" +prop_getBracedReference11b = getBracedReference "!os@" == "" +prop_getBracedReference12 = getBracedReference "!os?bar**" == "" +prop_getBracedReference13 = getBracedReference "foo[bar]" == "foo" getBracedReference s = fromMaybe s $ nameExpansion s `mplus` takeName noPrefix `mplus` getSpecial noPrefix `mplus` getSpecial s where diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 8b8d489..1429e1b 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -638,15 +638,15 @@ prop_checkShebang6 = verifyNotTree checkShebang "#!/usr/bin/env ash\n# shellchec prop_checkShebang7 = verifyNotTree checkShebang "#!/usr/bin/env ash\n# shellcheck shell=sh\n" prop_checkShebang8 = verifyTree checkShebang "#!bin/sh\ntrue" prop_checkShebang9 = verifyNotTree checkShebang "# shellcheck shell=sh\ntrue" -prop_checkShebang10= verifyNotTree checkShebang "#!foo\n# shellcheck shell=sh ignore=SC2239\ntrue" -prop_checkShebang11= verifyTree checkShebang "#!/bin/sh/\ntrue" -prop_checkShebang12= verifyTree checkShebang "#!/bin/sh/ -xe\ntrue" -prop_checkShebang13= verifyTree checkShebang "#!/bin/busybox sh" -prop_checkShebang14= verifyNotTree checkShebang "#!/bin/busybox sh\n# shellcheck shell=sh\n" -prop_checkShebang15= verifyNotTree checkShebang "#!/bin/busybox sh\n# shellcheck shell=dash\n" -prop_checkShebang16= verifyTree checkShebang "#!/bin/busybox ash" -prop_checkShebang17= verifyNotTree checkShebang "#!/bin/busybox ash\n# shellcheck shell=dash\n" -prop_checkShebang18= verifyNotTree checkShebang "#!/bin/busybox ash\n# shellcheck shell=sh\n" +prop_checkShebang10 = verifyNotTree checkShebang "#!foo\n# shellcheck shell=sh ignore=SC2239\ntrue" +prop_checkShebang11 = verifyTree checkShebang "#!/bin/sh/\ntrue" +prop_checkShebang12 = verifyTree checkShebang "#!/bin/sh/ -xe\ntrue" +prop_checkShebang13 = verifyTree checkShebang "#!/bin/busybox sh" +prop_checkShebang14 = verifyNotTree checkShebang "#!/bin/busybox sh\n# shellcheck shell=sh\n" +prop_checkShebang15 = verifyNotTree checkShebang "#!/bin/busybox sh\n# shellcheck shell=dash\n" +prop_checkShebang16 = verifyTree checkShebang "#!/bin/busybox ash" +prop_checkShebang17 = verifyNotTree checkShebang "#!/bin/busybox ash\n# shellcheck shell=dash\n" +prop_checkShebang18 = verifyNotTree checkShebang "#!/bin/busybox ash\n# shellcheck shell=sh\n" checkShebang params (T_Annotation _ list t) = if any isOverride list then [] else checkShebang params t where @@ -700,9 +700,9 @@ checkForInQuoted params (T_ForIn _ _ multiple _) = checkForInQuoted _ _ = return () prop_checkForInCat1 = verify checkForInCat "for f in $(cat foo); do stuff; done" -prop_checkForInCat1a= verify checkForInCat "for f in `cat foo`; do stuff; done" +prop_checkForInCat1a = verify checkForInCat "for f in `cat foo`; do stuff; done" prop_checkForInCat2 = verify checkForInCat "for f in $(cat foo | grep lol); do stuff; done" -prop_checkForInCat2a= verify checkForInCat "for f in `cat foo | grep lol`; do stuff; done" +prop_checkForInCat2a = verify checkForInCat "for f in `cat foo | grep lol`; do stuff; done" prop_checkForInCat3 = verifyNot checkForInCat "for f in $(cat foo | grep bar | wc -l); do stuff; done" checkForInCat _ (T_ForIn _ f [T_NormalWord _ w] _) = mapM_ checkF w where @@ -775,10 +775,10 @@ checkFindExec _ _ = return () prop_checkUnquotedExpansions1 = verify checkUnquotedExpansions "rm $(ls)" -prop_checkUnquotedExpansions1a= verify checkUnquotedExpansions "rm `ls`" +prop_checkUnquotedExpansions1a = verify checkUnquotedExpansions "rm `ls`" prop_checkUnquotedExpansions2 = verify checkUnquotedExpansions "rm foo$(date)" prop_checkUnquotedExpansions3 = verify checkUnquotedExpansions "[ $(foo) == cow ]" -prop_checkUnquotedExpansions3a= verify checkUnquotedExpansions "[ ! $(foo) ]" +prop_checkUnquotedExpansions3a = verify checkUnquotedExpansions "[ ! $(foo) ]" prop_checkUnquotedExpansions4 = verifyNot checkUnquotedExpansions "[[ $(foo) == cow ]]" prop_checkUnquotedExpansions5 = verifyNot checkUnquotedExpansions "for f in $(cmd); do echo $f; done" prop_checkUnquotedExpansions6 = verifyNot checkUnquotedExpansions "$(cmd)" @@ -906,7 +906,7 @@ checkDollarStar _ _ = return () prop_checkUnquotedDollarAt = verify checkUnquotedDollarAt "ls $@" -prop_checkUnquotedDollarAt1= verifyNot checkUnquotedDollarAt "ls ${#@}" +prop_checkUnquotedDollarAt1 = verifyNot checkUnquotedDollarAt "ls ${#@}" prop_checkUnquotedDollarAt2 = verify checkUnquotedDollarAt "ls ${foo[@]}" prop_checkUnquotedDollarAt3 = verifyNot checkUnquotedDollarAt "ls ${#foo[@]}" prop_checkUnquotedDollarAt4 = verifyNot checkUnquotedDollarAt "ls \"$@\"" @@ -1037,32 +1037,32 @@ ltt t = trace ("Tracing " ++ show t) -- STRIP prop_checkSingleQuotedVariables = verify checkSingleQuotedVariables "echo '$foo'" prop_checkSingleQuotedVariables2 = verify checkSingleQuotedVariables "echo 'lol$1.jpg'" prop_checkSingleQuotedVariables3 = verifyNot checkSingleQuotedVariables "sed 's/foo$/bar/'" -prop_checkSingleQuotedVariables3a= verify checkSingleQuotedVariables "sed 's/${foo}/bar/'" -prop_checkSingleQuotedVariables3b= verify checkSingleQuotedVariables "sed 's/$(echo cow)/bar/'" -prop_checkSingleQuotedVariables3c= verify checkSingleQuotedVariables "sed 's/$((1+foo))/bar/'" +prop_checkSingleQuotedVariables3a = verify checkSingleQuotedVariables "sed 's/${foo}/bar/'" +prop_checkSingleQuotedVariables3b = verify checkSingleQuotedVariables "sed 's/$(echo cow)/bar/'" +prop_checkSingleQuotedVariables3c = verify checkSingleQuotedVariables "sed 's/$((1+foo))/bar/'" prop_checkSingleQuotedVariables4 = verifyNot checkSingleQuotedVariables "awk '{print $1}'" prop_checkSingleQuotedVariables5 = verifyNot checkSingleQuotedVariables "trap 'echo $SECONDS' EXIT" prop_checkSingleQuotedVariables6 = verifyNot checkSingleQuotedVariables "sed -n '$p'" -prop_checkSingleQuotedVariables6a= verify checkSingleQuotedVariables "sed -n '$pattern'" +prop_checkSingleQuotedVariables6a = verify checkSingleQuotedVariables "sed -n '$pattern'" prop_checkSingleQuotedVariables7 = verifyNot checkSingleQuotedVariables "PS1='$PWD \\$ '" prop_checkSingleQuotedVariables8 = verify checkSingleQuotedVariables "find . -exec echo '$1' {} +" prop_checkSingleQuotedVariables9 = verifyNot checkSingleQuotedVariables "find . -exec awk '{print $1}' {} \\;" -prop_checkSingleQuotedVariables10= verify checkSingleQuotedVariables "echo '`pwd`'" -prop_checkSingleQuotedVariables11= verifyNot checkSingleQuotedVariables "sed '${/lol/d}'" -prop_checkSingleQuotedVariables12= verifyNot checkSingleQuotedVariables "eval 'echo $1'" -prop_checkSingleQuotedVariables13= verifyNot checkSingleQuotedVariables "busybox awk '{print $1}'" -prop_checkSingleQuotedVariables14= verifyNot checkSingleQuotedVariables "[ -v 'bar[$foo]' ]" -prop_checkSingleQuotedVariables15= verifyNot checkSingleQuotedVariables "git filter-branch 'test $GIT_COMMIT'" -prop_checkSingleQuotedVariables16= verify checkSingleQuotedVariables "git '$a'" -prop_checkSingleQuotedVariables17= verifyNot checkSingleQuotedVariables "rename 's/(.)a/$1/g' *" -prop_checkSingleQuotedVariables18= verifyNot checkSingleQuotedVariables "echo '``'" -prop_checkSingleQuotedVariables19= verifyNot checkSingleQuotedVariables "echo '```'" -prop_checkSingleQuotedVariables20= verifyNot checkSingleQuotedVariables "mumps -run %XCMD 'W $O(^GLOBAL(5))'" -prop_checkSingleQuotedVariables21= verifyNot checkSingleQuotedVariables "mumps -run LOOP%XCMD --xec 'W $O(^GLOBAL(6))'" -prop_checkSingleQuotedVariables22= verifyNot checkSingleQuotedVariables "jq '$__loc__'" -prop_checkSingleQuotedVariables23= verifyNot checkSingleQuotedVariables "command jq '$__loc__'" -prop_checkSingleQuotedVariables24= verifyNot checkSingleQuotedVariables "exec jq '$__loc__'" -prop_checkSingleQuotedVariables25= verifyNot checkSingleQuotedVariables "exec -c -a foo jq '$__loc__'" +prop_checkSingleQuotedVariables10 = verify checkSingleQuotedVariables "echo '`pwd`'" +prop_checkSingleQuotedVariables11 = verifyNot checkSingleQuotedVariables "sed '${/lol/d}'" +prop_checkSingleQuotedVariables12 = verifyNot checkSingleQuotedVariables "eval 'echo $1'" +prop_checkSingleQuotedVariables13 = verifyNot checkSingleQuotedVariables "busybox awk '{print $1}'" +prop_checkSingleQuotedVariables14 = verifyNot checkSingleQuotedVariables "[ -v 'bar[$foo]' ]" +prop_checkSingleQuotedVariables15 = verifyNot checkSingleQuotedVariables "git filter-branch 'test $GIT_COMMIT'" +prop_checkSingleQuotedVariables16 = verify checkSingleQuotedVariables "git '$a'" +prop_checkSingleQuotedVariables17 = verifyNot checkSingleQuotedVariables "rename 's/(.)a/$1/g' *" +prop_checkSingleQuotedVariables18 = verifyNot checkSingleQuotedVariables "echo '``'" +prop_checkSingleQuotedVariables19 = verifyNot checkSingleQuotedVariables "echo '```'" +prop_checkSingleQuotedVariables20 = verifyNot checkSingleQuotedVariables "mumps -run %XCMD 'W $O(^GLOBAL(5))'" +prop_checkSingleQuotedVariables21 = verifyNot checkSingleQuotedVariables "mumps -run LOOP%XCMD --xec 'W $O(^GLOBAL(6))'" +prop_checkSingleQuotedVariables22 = verifyNot checkSingleQuotedVariables "jq '$__loc__'" +prop_checkSingleQuotedVariables23 = verifyNot checkSingleQuotedVariables "command jq '$__loc__'" +prop_checkSingleQuotedVariables24 = verifyNot checkSingleQuotedVariables "exec jq '$__loc__'" +prop_checkSingleQuotedVariables25 = verifyNot checkSingleQuotedVariables "exec -c -a foo jq '$__loc__'" checkSingleQuotedVariables params t@(T_SingleQuoted id s) = @@ -1353,8 +1353,8 @@ checkGlobbedRegex _ _ = return () prop_checkConstantIfs1 = verify checkConstantIfs "[[ foo != bar ]]" -prop_checkConstantIfs2a= verify checkConstantIfs "[ n -le 4 ]" -prop_checkConstantIfs2b= verifyNot checkConstantIfs "[[ n -le 4 ]]" +prop_checkConstantIfs2a = verify checkConstantIfs "[ n -le 4 ]" +prop_checkConstantIfs2b = verifyNot checkConstantIfs "[[ n -le 4 ]]" prop_checkConstantIfs3 = verify checkConstantIfs "[[ $n -le 4 && n != 2 ]]" prop_checkConstantIfs4 = verifyNot checkConstantIfs "[[ $n -le 3 ]]" prop_checkConstantIfs5 = verifyNot checkConstantIfs "[[ $n -le $n ]]" @@ -1459,14 +1459,14 @@ prop_checkArithmeticDeref6 = verify checkArithmeticDeref "(( a[$i] ))" prop_checkArithmeticDeref7 = verifyNot checkArithmeticDeref "(( 10#$n ))" prop_checkArithmeticDeref8 = verifyNot checkArithmeticDeref "let i=$i+1" prop_checkArithmeticDeref9 = verifyNot checkArithmeticDeref "(( a[foo] ))" -prop_checkArithmeticDeref10= verifyNot checkArithmeticDeref "(( a[\\$foo] ))" -prop_checkArithmeticDeref11= verify checkArithmeticDeref "a[$foo]=wee" -prop_checkArithmeticDeref11b= verifyNot checkArithmeticDeref "declare -A a; a[$foo]=wee" -prop_checkArithmeticDeref12= verify checkArithmeticDeref "for ((i=0; $i < 3; i)); do true; done" -prop_checkArithmeticDeref13= verifyNot checkArithmeticDeref "(( $$ ))" -prop_checkArithmeticDeref14= verifyNot checkArithmeticDeref "(( $! ))" -prop_checkArithmeticDeref15= verifyNot checkArithmeticDeref "(( ${!var} ))" -prop_checkArithmeticDeref16= verifyNot checkArithmeticDeref "(( ${x+1} + ${x=42} ))" +prop_checkArithmeticDeref10 = verifyNot checkArithmeticDeref "(( a[\\$foo] ))" +prop_checkArithmeticDeref11 = verify checkArithmeticDeref "a[$foo]=wee" +prop_checkArithmeticDeref11b = verifyNot checkArithmeticDeref "declare -A a; a[$foo]=wee" +prop_checkArithmeticDeref12 = verify checkArithmeticDeref "for ((i=0; $i < 3; i)); do true; done" +prop_checkArithmeticDeref13 = verifyNot checkArithmeticDeref "(( $$ ))" +prop_checkArithmeticDeref14 = verifyNot checkArithmeticDeref "(( $! ))" +prop_checkArithmeticDeref15 = verifyNot checkArithmeticDeref "(( ${!var} ))" +prop_checkArithmeticDeref16 = verifyNot checkArithmeticDeref "(( ${x+1} + ${x=42} ))" checkArithmeticDeref params t@(TA_Expansion _ [T_DollarBraced id _ l]) = unless (isException $ concat $ oversimplify l) getWarning where @@ -1602,7 +1602,7 @@ checkOrNeq _ _ = return () prop_checkValidCondOps1 = verify checkValidCondOps "[[ a -xz b ]]" prop_checkValidCondOps2 = verify checkValidCondOps "[ -M a ]" -prop_checkValidCondOps2a= verifyNot checkValidCondOps "[ 3 \\> 2 ]" +prop_checkValidCondOps2a = verifyNot checkValidCondOps "[ 3 \\> 2 ]" prop_checkValidCondOps3 = verifyNot checkValidCondOps "[ 1 = 2 -a 3 -ge 4 ]" prop_checkValidCondOps4 = verifyNot checkValidCondOps "[[ ! -v foo ]]" checkValidCondOps _ (TC_Binary id _ s _ _) @@ -1670,11 +1670,11 @@ checkTestRedirects _ (T_Redirecting id redirs cmd) | cmd `isCommand` "test" = checkTestRedirects _ _ = return () prop_checkPS11 = verify checkPS1Assignments "PS1='\\033[1;35m\\$ '" -prop_checkPS11a= verify checkPS1Assignments "export PS1='\\033[1;35m\\$ '" +prop_checkPS11a = verify checkPS1Assignments "export PS1='\\033[1;35m\\$ '" prop_checkPSf2 = verify checkPS1Assignments "PS1='\\h \\e[0m\\$ '" prop_checkPS13 = verify checkPS1Assignments "PS1=$'\\x1b[c '" prop_checkPS14 = verify checkPS1Assignments "PS1=$'\\e[3m; '" -prop_checkPS14a= verify checkPS1Assignments "export PS1=$'\\e[3m; '" +prop_checkPS14a = verify checkPS1Assignments "export PS1=$'\\e[3m; '" prop_checkPS15 = verifyNot checkPS1Assignments "PS1='\\[\\033[1;35m\\]\\$ '" prop_checkPS16 = verifyNot checkPS1Assignments "PS1='\\[\\e1m\\e[1m\\]\\$ '" prop_checkPS17 = verifyNot checkPS1Assignments "PS1='e033x1B'" @@ -1943,7 +1943,7 @@ prop_subshellAssignmentCheck3 = verifyTree subshellAssignmentCheck "( A=foo; prop_subshellAssignmentCheck4 = verifyNotTree subshellAssignmentCheck "( A=foo; rm $A; )" prop_subshellAssignmentCheck5 = verifyTree subshellAssignmentCheck "cat foo | while read cow; do true; done; echo $cow;" prop_subshellAssignmentCheck6 = verifyTree subshellAssignmentCheck "( export lol=$(ls); ); echo $lol;" -prop_subshellAssignmentCheck6a= verifyTree subshellAssignmentCheck "( typeset -a lol=a; ); echo $lol;" +prop_subshellAssignmentCheck6a = verifyTree subshellAssignmentCheck "( typeset -a lol=a; ); echo $lol;" prop_subshellAssignmentCheck7 = verifyTree subshellAssignmentCheck "cmd | while read foo; do (( n++ )); done; echo \"$n lines\"" prop_subshellAssignmentCheck8 = verifyTree subshellAssignmentCheck "n=3 & echo $((n++))" prop_subshellAssignmentCheck9 = verifyTree subshellAssignmentCheck "read n & n=foo$n" @@ -2027,67 +2027,67 @@ prop_checkSpacefulnessCfg1 = verify checkSpacefulnessCfg "a='cow moo'; echo $a" prop_checkSpacefulnessCfg2 = verifyNot checkSpacefulnessCfg "a='cow moo'; [[ $a ]]" prop_checkSpacefulnessCfg3 = verifyNot checkSpacefulnessCfg "a='cow*.mp3'; echo \"$a\"" prop_checkSpacefulnessCfg4 = verify checkSpacefulnessCfg "for f in *.mp3; do echo $f; done" -prop_checkSpacefulnessCfg4a= verifyNot checkSpacefulnessCfg "foo=3; foo=$(echo $foo)" +prop_checkSpacefulnessCfg4a = verifyNot checkSpacefulnessCfg "foo=3; foo=$(echo $foo)" prop_checkSpacefulnessCfg5 = verify checkSpacefulnessCfg "a='*'; b=$a; c=lol${b//foo/bar}; echo $c" prop_checkSpacefulnessCfg6 = verify checkSpacefulnessCfg "a=foo$(lol); echo $a" prop_checkSpacefulnessCfg7 = verify checkSpacefulnessCfg "a=foo\\ bar; rm $a" prop_checkSpacefulnessCfg8 = verifyNot checkSpacefulnessCfg "a=foo\\ bar; a=foo; rm $a" -prop_checkSpacefulnessCfg10= verify checkSpacefulnessCfg "rm $1" -prop_checkSpacefulnessCfg11= verify checkSpacefulnessCfg "rm ${10//foo/bar}" -prop_checkSpacefulnessCfg12= verifyNot checkSpacefulnessCfg "(( $1 + 3 ))" -prop_checkSpacefulnessCfg13= verifyNot checkSpacefulnessCfg "if [[ $2 -gt 14 ]]; then true; fi" -prop_checkSpacefulnessCfg14= verifyNot checkSpacefulnessCfg "foo=$3 env" -prop_checkSpacefulnessCfg15= verifyNot checkSpacefulnessCfg "local foo=$1" -prop_checkSpacefulnessCfg16= verifyNot checkSpacefulnessCfg "declare foo=$1" -prop_checkSpacefulnessCfg17= verify checkSpacefulnessCfg "echo foo=$1" -prop_checkSpacefulnessCfg18= verifyNot checkSpacefulnessCfg "$1 --flags" -prop_checkSpacefulnessCfg19= verify checkSpacefulnessCfg "echo $PWD" -prop_checkSpacefulnessCfg20= verifyNot checkSpacefulnessCfg "n+='foo bar'" -prop_checkSpacefulnessCfg21= verifyNot checkSpacefulnessCfg "select foo in $bar; do true; done" -prop_checkSpacefulnessCfg22= verifyNot checkSpacefulnessCfg "echo $\"$1\"" -prop_checkSpacefulnessCfg23= verifyNot checkSpacefulnessCfg "a=(1); echo ${a[@]}" -prop_checkSpacefulnessCfg24= verify checkSpacefulnessCfg "a='a b'; cat <<< $a" -prop_checkSpacefulnessCfg25= verify checkSpacefulnessCfg "a='s/[0-9]//g'; sed $a" -prop_checkSpacefulnessCfg26= verify checkSpacefulnessCfg "a='foo bar'; echo {1,2,$a}" -prop_checkSpacefulnessCfg27= verifyNot checkSpacefulnessCfg "echo ${a:+'foo'}" -prop_checkSpacefulnessCfg28= verifyNot checkSpacefulnessCfg "exec {n}>&1; echo $n" -prop_checkSpacefulnessCfg29= verifyNot checkSpacefulnessCfg "n=$(stuff); exec {n}>&-;" -prop_checkSpacefulnessCfg30= verify checkSpacefulnessCfg "file='foo bar'; echo foo > $file;" -prop_checkSpacefulnessCfg31= verifyNot checkSpacefulnessCfg "echo \"`echo \\\"$1\\\"`\"" -prop_checkSpacefulnessCfg32= verifyNot checkSpacefulnessCfg "var=$1; [ -v var ]" -prop_checkSpacefulnessCfg33= verify checkSpacefulnessCfg "for file; do echo $file; done" -prop_checkSpacefulnessCfg34= verify checkSpacefulnessCfg "declare foo$n=$1" -prop_checkSpacefulnessCfg35= verifyNot checkSpacefulnessCfg "echo ${1+\"$1\"}" -prop_checkSpacefulnessCfg36= verifyNot checkSpacefulnessCfg "arg=$#; echo $arg" -prop_checkSpacefulnessCfg37= verifyNot checkSpacefulnessCfg "@test 'status' {\n [ $status -eq 0 ]\n}" +prop_checkSpacefulnessCfg10 = verify checkSpacefulnessCfg "rm $1" +prop_checkSpacefulnessCfg11 = verify checkSpacefulnessCfg "rm ${10//foo/bar}" +prop_checkSpacefulnessCfg12 = verifyNot checkSpacefulnessCfg "(( $1 + 3 ))" +prop_checkSpacefulnessCfg13 = verifyNot checkSpacefulnessCfg "if [[ $2 -gt 14 ]]; then true; fi" +prop_checkSpacefulnessCfg14 = verifyNot checkSpacefulnessCfg "foo=$3 env" +prop_checkSpacefulnessCfg15 = verifyNot checkSpacefulnessCfg "local foo=$1" +prop_checkSpacefulnessCfg16 = verifyNot checkSpacefulnessCfg "declare foo=$1" +prop_checkSpacefulnessCfg17 = verify checkSpacefulnessCfg "echo foo=$1" +prop_checkSpacefulnessCfg18 = verifyNot checkSpacefulnessCfg "$1 --flags" +prop_checkSpacefulnessCfg19 = verify checkSpacefulnessCfg "echo $PWD" +prop_checkSpacefulnessCfg20 = verifyNot checkSpacefulnessCfg "n+='foo bar'" +prop_checkSpacefulnessCfg21 = verifyNot checkSpacefulnessCfg "select foo in $bar; do true; done" +prop_checkSpacefulnessCfg22 = verifyNot checkSpacefulnessCfg "echo $\"$1\"" +prop_checkSpacefulnessCfg23 = verifyNot checkSpacefulnessCfg "a=(1); echo ${a[@]}" +prop_checkSpacefulnessCfg24 = verify checkSpacefulnessCfg "a='a b'; cat <<< $a" +prop_checkSpacefulnessCfg25 = verify checkSpacefulnessCfg "a='s/[0-9]//g'; sed $a" +prop_checkSpacefulnessCfg26 = verify checkSpacefulnessCfg "a='foo bar'; echo {1,2,$a}" +prop_checkSpacefulnessCfg27 = verifyNot checkSpacefulnessCfg "echo ${a:+'foo'}" +prop_checkSpacefulnessCfg28 = verifyNot checkSpacefulnessCfg "exec {n}>&1; echo $n" +prop_checkSpacefulnessCfg29 = verifyNot checkSpacefulnessCfg "n=$(stuff); exec {n}>&-;" +prop_checkSpacefulnessCfg30 = verify checkSpacefulnessCfg "file='foo bar'; echo foo > $file;" +prop_checkSpacefulnessCfg31 = verifyNot checkSpacefulnessCfg "echo \"`echo \\\"$1\\\"`\"" +prop_checkSpacefulnessCfg32 = verifyNot checkSpacefulnessCfg "var=$1; [ -v var ]" +prop_checkSpacefulnessCfg33 = verify checkSpacefulnessCfg "for file; do echo $file; done" +prop_checkSpacefulnessCfg34 = verify checkSpacefulnessCfg "declare foo$n=$1" +prop_checkSpacefulnessCfg35 = verifyNot checkSpacefulnessCfg "echo ${1+\"$1\"}" +prop_checkSpacefulnessCfg36 = verifyNot checkSpacefulnessCfg "arg=$#; echo $arg" +prop_checkSpacefulnessCfg37 = verifyNot checkSpacefulnessCfg "@test 'status' {\n [ $status -eq 0 ]\n}" prop_checkSpacefulnessCfg37v = verify checkVerboseSpacefulnessCfg "@test 'status' {\n [ $status -eq 0 ]\n}" -prop_checkSpacefulnessCfg38= verify checkSpacefulnessCfg "a=; echo $a" -prop_checkSpacefulnessCfg39= verifyNot checkSpacefulnessCfg "a=''\"\"''; b=x$a; echo $b" -prop_checkSpacefulnessCfg40= verifyNot checkSpacefulnessCfg "a=$((x+1)); echo $a" -prop_checkSpacefulnessCfg41= verifyNot checkSpacefulnessCfg "exec $1 --flags" -prop_checkSpacefulnessCfg42= verifyNot checkSpacefulnessCfg "run $1 --flags" -prop_checkSpacefulnessCfg43= verifyNot checkSpacefulnessCfg "$foo=42" -prop_checkSpacefulnessCfg44= verify checkSpacefulnessCfg "#!/bin/sh\nexport var=$value" -prop_checkSpacefulnessCfg45= verifyNot checkSpacefulnessCfg "wait -zzx -p foo; echo $foo" -prop_checkSpacefulnessCfg46= verifyNot checkSpacefulnessCfg "x=0; (( x += 1 )); echo $x" -prop_checkSpacefulnessCfg47= verifyNot checkSpacefulnessCfg "x=0; (( x-- )); echo $x" -prop_checkSpacefulnessCfg48= verifyNot checkSpacefulnessCfg "x=0; (( ++x )); echo $x" -prop_checkSpacefulnessCfg49= verifyNot checkSpacefulnessCfg "for i in 1 2 3; do echo $i; done" -prop_checkSpacefulnessCfg50= verify checkSpacefulnessCfg "for i in 1 2 *; do echo $i; done" -prop_checkSpacefulnessCfg51= verify checkSpacefulnessCfg "x='foo bar'; x && x=1; echo $x" -prop_checkSpacefulnessCfg52= verifyNot checkSpacefulnessCfg "x=1; if f; then x='foo bar'; exit; fi; echo $x" -prop_checkSpacefulnessCfg53= verifyNot checkSpacefulnessCfg "s=1; f() { local s='a b'; }; f; echo $s" -prop_checkSpacefulnessCfg54= verifyNot checkSpacefulnessCfg "s='a b'; f() { s=1; }; f; echo $s" -prop_checkSpacefulnessCfg55= verify checkSpacefulnessCfg "s='a b'; x && f() { s=1; }; f; echo $s" -prop_checkSpacefulnessCfg56= verifyNot checkSpacefulnessCfg "s=1; cat <(s='a b'); echo $s" -prop_checkSpacefulnessCfg57= verifyNot checkSpacefulnessCfg "declare -i s=0; s=$(f); echo $s" -prop_checkSpacefulnessCfg58= verify checkSpacefulnessCfg "f() { declare -i s; }; f; s=$(var); echo $s" -prop_checkSpacefulnessCfg59= verifyNot checkSpacefulnessCfg "f() { declare -gi s; }; f; s=$(var); echo $s" -prop_checkSpacefulnessCfg60= verify checkSpacefulnessCfg "declare -i s; declare +i s; s=$(foo); echo $s" -prop_checkSpacefulnessCfg61= verify checkSpacefulnessCfg "declare -x X; y=foo$X; echo $y;" -prop_checkSpacefulnessCfg62= verifyNot checkSpacefulnessCfg "f() { declare -x X; y=foo$X; echo $y; }" -prop_checkSpacefulnessCfg63= verify checkSpacefulnessCfg "f && declare -i s; s='x + y'; echo $s" -prop_checkSpacefulnessCfg64= verifyNot checkSpacefulnessCfg "declare -i s; s='x + y'; x=$s; echo $x" +prop_checkSpacefulnessCfg38 = verify checkSpacefulnessCfg "a=; echo $a" +prop_checkSpacefulnessCfg39 = verifyNot checkSpacefulnessCfg "a=''\"\"''; b=x$a; echo $b" +prop_checkSpacefulnessCfg40 = verifyNot checkSpacefulnessCfg "a=$((x+1)); echo $a" +prop_checkSpacefulnessCfg41 = verifyNot checkSpacefulnessCfg "exec $1 --flags" +prop_checkSpacefulnessCfg42 = verifyNot checkSpacefulnessCfg "run $1 --flags" +prop_checkSpacefulnessCfg43 = verifyNot checkSpacefulnessCfg "$foo=42" +prop_checkSpacefulnessCfg44 = verify checkSpacefulnessCfg "#!/bin/sh\nexport var=$value" +prop_checkSpacefulnessCfg45 = verifyNot checkSpacefulnessCfg "wait -zzx -p foo; echo $foo" +prop_checkSpacefulnessCfg46 = verifyNot checkSpacefulnessCfg "x=0; (( x += 1 )); echo $x" +prop_checkSpacefulnessCfg47 = verifyNot checkSpacefulnessCfg "x=0; (( x-- )); echo $x" +prop_checkSpacefulnessCfg48 = verifyNot checkSpacefulnessCfg "x=0; (( ++x )); echo $x" +prop_checkSpacefulnessCfg49 = verifyNot checkSpacefulnessCfg "for i in 1 2 3; do echo $i; done" +prop_checkSpacefulnessCfg50 = verify checkSpacefulnessCfg "for i in 1 2 *; do echo $i; done" +prop_checkSpacefulnessCfg51 = verify checkSpacefulnessCfg "x='foo bar'; x && x=1; echo $x" +prop_checkSpacefulnessCfg52 = verifyNot checkSpacefulnessCfg "x=1; if f; then x='foo bar'; exit; fi; echo $x" +prop_checkSpacefulnessCfg53 = verifyNot checkSpacefulnessCfg "s=1; f() { local s='a b'; }; f; echo $s" +prop_checkSpacefulnessCfg54 = verifyNot checkSpacefulnessCfg "s='a b'; f() { s=1; }; f; echo $s" +prop_checkSpacefulnessCfg55 = verify checkSpacefulnessCfg "s='a b'; x && f() { s=1; }; f; echo $s" +prop_checkSpacefulnessCfg56 = verifyNot checkSpacefulnessCfg "s=1; cat <(s='a b'); echo $s" +prop_checkSpacefulnessCfg57 = verifyNot checkSpacefulnessCfg "declare -i s=0; s=$(f); echo $s" +prop_checkSpacefulnessCfg58 = verify checkSpacefulnessCfg "f() { declare -i s; }; f; s=$(var); echo $s" +prop_checkSpacefulnessCfg59 = verifyNot checkSpacefulnessCfg "f() { declare -gi s; }; f; s=$(var); echo $s" +prop_checkSpacefulnessCfg60 = verify checkSpacefulnessCfg "declare -i s; declare +i s; s=$(foo); echo $s" +prop_checkSpacefulnessCfg61 = verify checkSpacefulnessCfg "declare -x X; y=foo$X; echo $y;" +prop_checkSpacefulnessCfg62 = verifyNot checkSpacefulnessCfg "f() { declare -x X; y=foo$X; echo $y; }" +prop_checkSpacefulnessCfg63 = verify checkSpacefulnessCfg "f && declare -i s; s='x + y'; echo $s" +prop_checkSpacefulnessCfg64 = verifyNot checkSpacefulnessCfg "declare -i s; s='x + y'; x=$s; echo $x" checkSpacefulnessCfg = checkSpacefulnessCfg' True checkVerboseSpacefulnessCfg = checkSpacefulnessCfg' False @@ -2157,13 +2157,13 @@ checkVariableBraces params t@(T_DollarBraced id False l) checkVariableBraces _ _ = return () prop_checkQuotesInLiterals1 = verifyTree checkQuotesInLiterals "param='--foo=\"bar\"'; app $param" -prop_checkQuotesInLiterals1a= verifyTree checkQuotesInLiterals "param=\"--foo='lolbar'\"; app $param" +prop_checkQuotesInLiterals1a = verifyTree checkQuotesInLiterals "param=\"--foo='lolbar'\"; app $param" prop_checkQuotesInLiterals2 = verifyNotTree checkQuotesInLiterals "param='--foo=\"bar\"'; app \"$param\"" prop_checkQuotesInLiterals3 =verifyNotTree checkQuotesInLiterals "param=('--foo='); app \"${param[@]}\"" prop_checkQuotesInLiterals4 = verifyNotTree checkQuotesInLiterals "param=\"don't bother with this one\"; app $param" prop_checkQuotesInLiterals5 = verifyNotTree checkQuotesInLiterals "param=\"--foo='lolbar'\"; eval app $param" prop_checkQuotesInLiterals6 = verifyTree checkQuotesInLiterals "param='my\\ file'; cmd=\"rm $param\"; $cmd" -prop_checkQuotesInLiterals6a= verifyNotTree checkQuotesInLiterals "param='my\\ file'; cmd=\"rm ${#param}\"; $cmd" +prop_checkQuotesInLiterals6a = verifyNotTree checkQuotesInLiterals "param='my\\ file'; cmd=\"rm ${#param}\"; $cmd" prop_checkQuotesInLiterals7 = verifyTree checkQuotesInLiterals "param='my\\ file'; rm $param" prop_checkQuotesInLiterals8 = verifyTree checkQuotesInLiterals "param=\"/foo/'bar baz'/etc\"; rm $param" prop_checkQuotesInLiterals9 = verifyNotTree checkQuotesInLiterals "param=\"/foo/'bar baz'/etc\"; rm ${#param}" @@ -2227,9 +2227,9 @@ prop_checkFunctionsUsedExternally1 = verifyTree checkFunctionsUsedExternally "foo() { :; }; sudo foo" prop_checkFunctionsUsedExternally2 = verifyTree checkFunctionsUsedExternally "alias f='a'; xargs -0 f" -prop_checkFunctionsUsedExternally2b= +prop_checkFunctionsUsedExternally2b = verifyNotTree checkFunctionsUsedExternally "alias f='a'; find . -type f" -prop_checkFunctionsUsedExternally2c= +prop_checkFunctionsUsedExternally2c = verifyTree checkFunctionsUsedExternally "alias f='a'; find . -type f -exec f +" prop_checkFunctionsUsedExternally3 = verifyNotTree checkFunctionsUsedExternally "f() { :; }; echo f" @@ -2305,49 +2305,49 @@ prop_checkUnused6 = verifyNotTree checkUnusedAssignments "var=4; (( var++ ))" prop_checkUnused7 = verifyNotTree checkUnusedAssignments "var=2; $((var))" prop_checkUnused8 = verifyTree checkUnusedAssignments "var=2; var=3;" prop_checkUnused9 = verifyNotTree checkUnusedAssignments "read ''" -prop_checkUnused10= verifyNotTree checkUnusedAssignments "read -p 'test: '" -prop_checkUnused11= verifyNotTree checkUnusedAssignments "bar=5; export foo[$bar]=3" -prop_checkUnused12= verifyNotTree checkUnusedAssignments "read foo; echo ${!foo}" -prop_checkUnused13= verifyNotTree checkUnusedAssignments "x=(1); (( x[0] ))" -prop_checkUnused14= verifyNotTree checkUnusedAssignments "x=(1); n=0; echo ${x[n]}" -prop_checkUnused15= verifyNotTree checkUnusedAssignments "x=(1); n=0; (( x[n] ))" -prop_checkUnused16= verifyNotTree checkUnusedAssignments "foo=5; declare -x foo" -prop_checkUnused16b= verifyNotTree checkUnusedAssignments "f() { local -x foo; foo=42; bar; }; f" -prop_checkUnused17= verifyNotTree checkUnusedAssignments "read -i 'foo' -e -p 'Input: ' bar; $bar;" -prop_checkUnused18= verifyNotTree checkUnusedAssignments "a=1; arr=( [$a]=42 ); echo \"${arr[@]}\"" -prop_checkUnused19= verifyNotTree checkUnusedAssignments "a=1; let b=a+1; echo $b" -prop_checkUnused20= verifyNotTree checkUnusedAssignments "a=1; PS1='$a'" -prop_checkUnused21= verifyNotTree checkUnusedAssignments "a=1; trap 'echo $a' INT" -prop_checkUnused22= verifyNotTree checkUnusedAssignments "a=1; [ -v a ]" -prop_checkUnused23= verifyNotTree checkUnusedAssignments "a=1; [ -R a ]" -prop_checkUnused24= verifyNotTree checkUnusedAssignments "mapfile -C a b; echo ${b[@]}" -prop_checkUnused25= verifyNotTree checkUnusedAssignments "readarray foo; echo ${foo[@]}" -prop_checkUnused26= verifyNotTree checkUnusedAssignments "declare -F foo" -prop_checkUnused27= verifyTree checkUnusedAssignments "var=3; [ var -eq 3 ]" -prop_checkUnused28= verifyNotTree checkUnusedAssignments "var=3; [[ var -eq 3 ]]" -prop_checkUnused29= verifyNotTree checkUnusedAssignments "var=(a b); declare -p var" -prop_checkUnused30= verifyTree checkUnusedAssignments "let a=1" -prop_checkUnused31= verifyTree checkUnusedAssignments "let 'a=1'" -prop_checkUnused32= verifyTree checkUnusedAssignments "let a=b=c; echo $a" -prop_checkUnused33= verifyNotTree checkUnusedAssignments "a=foo; [[ foo =~ ^{$a}$ ]]" -prop_checkUnused34= verifyNotTree checkUnusedAssignments "foo=1; (( t = foo )); echo $t" -prop_checkUnused35= verifyNotTree checkUnusedAssignments "a=foo; b=2; echo ${a:b}" -prop_checkUnused36= verifyNotTree checkUnusedAssignments "if [[ -v foo ]]; then true; fi" -prop_checkUnused37= verifyNotTree checkUnusedAssignments "fd=2; exec {fd}>&-" -prop_checkUnused38= verifyTree checkUnusedAssignments "(( a=42 ))" -prop_checkUnused39= verifyNotTree checkUnusedAssignments "declare -x -f foo" -prop_checkUnused40= verifyNotTree checkUnusedAssignments "arr=(1 2); num=2; echo \"${arr[@]:num}\"" -prop_checkUnused41= verifyNotTree checkUnusedAssignments "@test 'foo' {\ntrue\n}\n" -prop_checkUnused42= verifyNotTree checkUnusedAssignments "DEFINE_string foo '' ''; echo \"${FLAGS_foo}\"" -prop_checkUnused43= verifyTree checkUnusedAssignments "DEFINE_string foo '' ''" -prop_checkUnused44= verifyNotTree checkUnusedAssignments "DEFINE_string \"foo$ibar\" x y" -prop_checkUnused45= verifyTree checkUnusedAssignments "readonly foo=bar" -prop_checkUnused46= verifyTree checkUnusedAssignments "readonly foo=(bar)" -prop_checkUnused47= verifyNotTree checkUnusedAssignments "a=1; alias hello='echo $a'" -prop_checkUnused48= verifyNotTree checkUnusedAssignments "_a=1" -prop_checkUnused49= verifyNotTree checkUnusedAssignments "declare -A array; key=a; [[ -v array[$key] ]]" -prop_checkUnused50= verifyNotTree checkUnusedAssignments "foofunc() { :; }; typeset -fx foofunc" -prop_checkUnused51= verifyTree checkUnusedAssignments "x[y[z=1]]=1; echo ${x[@]}" +prop_checkUnused10 = verifyNotTree checkUnusedAssignments "read -p 'test: '" +prop_checkUnused11 = verifyNotTree checkUnusedAssignments "bar=5; export foo[$bar]=3" +prop_checkUnused12 = verifyNotTree checkUnusedAssignments "read foo; echo ${!foo}" +prop_checkUnused13 = verifyNotTree checkUnusedAssignments "x=(1); (( x[0] ))" +prop_checkUnused14 = verifyNotTree checkUnusedAssignments "x=(1); n=0; echo ${x[n]}" +prop_checkUnused15 = verifyNotTree checkUnusedAssignments "x=(1); n=0; (( x[n] ))" +prop_checkUnused16 = verifyNotTree checkUnusedAssignments "foo=5; declare -x foo" +prop_checkUnused16b = verifyNotTree checkUnusedAssignments "f() { local -x foo; foo=42; bar; }; f" +prop_checkUnused17 = verifyNotTree checkUnusedAssignments "read -i 'foo' -e -p 'Input: ' bar; $bar;" +prop_checkUnused18 = verifyNotTree checkUnusedAssignments "a=1; arr=( [$a]=42 ); echo \"${arr[@]}\"" +prop_checkUnused19 = verifyNotTree checkUnusedAssignments "a=1; let b=a+1; echo $b" +prop_checkUnused20 = verifyNotTree checkUnusedAssignments "a=1; PS1='$a'" +prop_checkUnused21 = verifyNotTree checkUnusedAssignments "a=1; trap 'echo $a' INT" +prop_checkUnused22 = verifyNotTree checkUnusedAssignments "a=1; [ -v a ]" +prop_checkUnused23 = verifyNotTree checkUnusedAssignments "a=1; [ -R a ]" +prop_checkUnused24 = verifyNotTree checkUnusedAssignments "mapfile -C a b; echo ${b[@]}" +prop_checkUnused25 = verifyNotTree checkUnusedAssignments "readarray foo; echo ${foo[@]}" +prop_checkUnused26 = verifyNotTree checkUnusedAssignments "declare -F foo" +prop_checkUnused27 = verifyTree checkUnusedAssignments "var=3; [ var -eq 3 ]" +prop_checkUnused28 = verifyNotTree checkUnusedAssignments "var=3; [[ var -eq 3 ]]" +prop_checkUnused29 = verifyNotTree checkUnusedAssignments "var=(a b); declare -p var" +prop_checkUnused30 = verifyTree checkUnusedAssignments "let a=1" +prop_checkUnused31 = verifyTree checkUnusedAssignments "let 'a=1'" +prop_checkUnused32 = verifyTree checkUnusedAssignments "let a=b=c; echo $a" +prop_checkUnused33 = verifyNotTree checkUnusedAssignments "a=foo; [[ foo =~ ^{$a}$ ]]" +prop_checkUnused34 = verifyNotTree checkUnusedAssignments "foo=1; (( t = foo )); echo $t" +prop_checkUnused35 = verifyNotTree checkUnusedAssignments "a=foo; b=2; echo ${a:b}" +prop_checkUnused36 = verifyNotTree checkUnusedAssignments "if [[ -v foo ]]; then true; fi" +prop_checkUnused37 = verifyNotTree checkUnusedAssignments "fd=2; exec {fd}>&-" +prop_checkUnused38 = verifyTree checkUnusedAssignments "(( a=42 ))" +prop_checkUnused39 = verifyNotTree checkUnusedAssignments "declare -x -f foo" +prop_checkUnused40 = verifyNotTree checkUnusedAssignments "arr=(1 2); num=2; echo \"${arr[@]:num}\"" +prop_checkUnused41 = verifyNotTree checkUnusedAssignments "@test 'foo' {\ntrue\n}\n" +prop_checkUnused42 = verifyNotTree checkUnusedAssignments "DEFINE_string foo '' ''; echo \"${FLAGS_foo}\"" +prop_checkUnused43 = verifyTree checkUnusedAssignments "DEFINE_string foo '' ''" +prop_checkUnused44 = verifyNotTree checkUnusedAssignments "DEFINE_string \"foo$ibar\" x y" +prop_checkUnused45 = verifyTree checkUnusedAssignments "readonly foo=bar" +prop_checkUnused46 = verifyTree checkUnusedAssignments "readonly foo=(bar)" +prop_checkUnused47 = verifyNotTree checkUnusedAssignments "a=1; alias hello='echo $a'" +prop_checkUnused48 = verifyNotTree checkUnusedAssignments "_a=1" +prop_checkUnused49 = verifyNotTree checkUnusedAssignments "declare -A array; key=a; [[ -v array[$key] ]]" +prop_checkUnused50 = verifyNotTree checkUnusedAssignments "foofunc() { :; }; typeset -fx foofunc" +prop_checkUnused51 = verifyTree checkUnusedAssignments "x[y[z=1]]=1; echo ${x[@]}" checkUnusedAssignments params t = execWriter (mapM_ warnFor unused) where @@ -2381,40 +2381,40 @@ prop_checkUnassignedReferences6 = verifyNotTree checkUnassignedReferences "foo=. prop_checkUnassignedReferences7 = verifyNotTree checkUnassignedReferences "getopts ':h' foo; echo $foo" prop_checkUnassignedReferences8 = verifyNotTree checkUnassignedReferences "let 'foo = 1'; echo $foo" prop_checkUnassignedReferences9 = verifyNotTree checkUnassignedReferences "echo ${foo-bar}" -prop_checkUnassignedReferences10= verifyNotTree checkUnassignedReferences "echo ${foo:?}" -prop_checkUnassignedReferences11= verifyNotTree checkUnassignedReferences "declare -A foo; echo \"${foo[@]}\"" -prop_checkUnassignedReferences12= verifyNotTree checkUnassignedReferences "typeset -a foo; echo \"${foo[@]}\"" -prop_checkUnassignedReferences13= verifyNotTree checkUnassignedReferences "f() { local foo; echo $foo; }" -prop_checkUnassignedReferences14= verifyNotTree checkUnassignedReferences "foo=; echo $foo" -prop_checkUnassignedReferences15= verifyNotTree checkUnassignedReferences "f() { true; }; export -f f" -prop_checkUnassignedReferences16= verifyNotTree checkUnassignedReferences "declare -A foo=( [a b]=bar ); echo ${foo[a b]}" -prop_checkUnassignedReferences17= verifyNotTree checkUnassignedReferences "USERS=foo; echo $USER" -prop_checkUnassignedReferences18= verifyNotTree checkUnassignedReferences "FOOBAR=42; export FOOBAR=" -prop_checkUnassignedReferences19= verifyNotTree checkUnassignedReferences "readonly foo=bar; echo $foo" -prop_checkUnassignedReferences20= verifyNotTree checkUnassignedReferences "printf -v foo bar; echo $foo" -prop_checkUnassignedReferences21= verifyTree checkUnassignedReferences "echo ${#foo}" -prop_checkUnassignedReferences22= verifyNotTree checkUnassignedReferences "echo ${!os*}" -prop_checkUnassignedReferences23= verifyTree checkUnassignedReferences "declare -a foo; foo[bar]=42;" -prop_checkUnassignedReferences24= verifyNotTree checkUnassignedReferences "declare -A foo; foo[bar]=42;" -prop_checkUnassignedReferences25= verifyNotTree checkUnassignedReferences "declare -A foo=(); foo[bar]=42;" -prop_checkUnassignedReferences26= verifyNotTree checkUnassignedReferences "a::b() { foo; }; readonly -f a::b" -prop_checkUnassignedReferences27= verifyNotTree checkUnassignedReferences ": ${foo:=bar}" -prop_checkUnassignedReferences28= verifyNotTree checkUnassignedReferences "#!/bin/ksh\necho \"${.sh.version}\"\n" -prop_checkUnassignedReferences29= verifyNotTree checkUnassignedReferences "if [[ -v foo ]]; then echo $foo; fi" -prop_checkUnassignedReferences30= verifyNotTree checkUnassignedReferences "if [[ -v foo[3] ]]; then echo ${foo[3]}; fi" -prop_checkUnassignedReferences31= verifyNotTree checkUnassignedReferences "X=1; if [[ -v foo[$X+42] ]]; then echo ${foo[$X+42]}; fi" -prop_checkUnassignedReferences32= verifyNotTree checkUnassignedReferences "if [[ -v \"foo[1]\" ]]; then echo ${foo[@]}; fi" -prop_checkUnassignedReferences33= verifyNotTree checkUnassignedReferences "f() { local -A foo; echo \"${foo[@]}\"; }" -prop_checkUnassignedReferences34= verifyNotTree checkUnassignedReferences "declare -A foo; (( foo[bar] ))" -prop_checkUnassignedReferences35= verifyNotTree checkUnassignedReferences "echo ${arr[foo-bar]:?fail}" -prop_checkUnassignedReferences36= verifyNotTree checkUnassignedReferences "read -a foo -r <<<\"foo bar\"; echo \"$foo\"" -prop_checkUnassignedReferences37= verifyNotTree checkUnassignedReferences "var=howdy; printf -v 'array[0]' %s \"$var\"; printf %s \"${array[0]}\";" -prop_checkUnassignedReferences38= verifyTree (checkUnassignedReferences' True) "echo $VAR" -prop_checkUnassignedReferences39= verifyNotTree checkUnassignedReferences "builtin export var=4; echo $var" -prop_checkUnassignedReferences40= verifyNotTree checkUnassignedReferences ": ${foo=bar}" -prop_checkUnassignedReferences41= verifyNotTree checkUnassignedReferences "mapfile -t files 123; echo \"${files[@]}\"" -prop_checkUnassignedReferences42= verifyNotTree checkUnassignedReferences "mapfile files -t; echo \"${files[@]}\"" -prop_checkUnassignedReferences43= verifyNotTree checkUnassignedReferences "mapfile --future files; echo \"${files[@]}\"" +prop_checkUnassignedReferences10 = verifyNotTree checkUnassignedReferences "echo ${foo:?}" +prop_checkUnassignedReferences11 = verifyNotTree checkUnassignedReferences "declare -A foo; echo \"${foo[@]}\"" +prop_checkUnassignedReferences12 = verifyNotTree checkUnassignedReferences "typeset -a foo; echo \"${foo[@]}\"" +prop_checkUnassignedReferences13 = verifyNotTree checkUnassignedReferences "f() { local foo; echo $foo; }" +prop_checkUnassignedReferences14 = verifyNotTree checkUnassignedReferences "foo=; echo $foo" +prop_checkUnassignedReferences15 = verifyNotTree checkUnassignedReferences "f() { true; }; export -f f" +prop_checkUnassignedReferences16 = verifyNotTree checkUnassignedReferences "declare -A foo=( [a b]=bar ); echo ${foo[a b]}" +prop_checkUnassignedReferences17 = verifyNotTree checkUnassignedReferences "USERS=foo; echo $USER" +prop_checkUnassignedReferences18 = verifyNotTree checkUnassignedReferences "FOOBAR=42; export FOOBAR=" +prop_checkUnassignedReferences19 = verifyNotTree checkUnassignedReferences "readonly foo=bar; echo $foo" +prop_checkUnassignedReferences20 = verifyNotTree checkUnassignedReferences "printf -v foo bar; echo $foo" +prop_checkUnassignedReferences21 = verifyTree checkUnassignedReferences "echo ${#foo}" +prop_checkUnassignedReferences22 = verifyNotTree checkUnassignedReferences "echo ${!os*}" +prop_checkUnassignedReferences23 = verifyTree checkUnassignedReferences "declare -a foo; foo[bar]=42;" +prop_checkUnassignedReferences24 = verifyNotTree checkUnassignedReferences "declare -A foo; foo[bar]=42;" +prop_checkUnassignedReferences25 = verifyNotTree checkUnassignedReferences "declare -A foo=(); foo[bar]=42;" +prop_checkUnassignedReferences26 = verifyNotTree checkUnassignedReferences "a::b() { foo; }; readonly -f a::b" +prop_checkUnassignedReferences27 = verifyNotTree checkUnassignedReferences ": ${foo:=bar}" +prop_checkUnassignedReferences28 = verifyNotTree checkUnassignedReferences "#!/bin/ksh\necho \"${.sh.version}\"\n" +prop_checkUnassignedReferences29 = verifyNotTree checkUnassignedReferences "if [[ -v foo ]]; then echo $foo; fi" +prop_checkUnassignedReferences30 = verifyNotTree checkUnassignedReferences "if [[ -v foo[3] ]]; then echo ${foo[3]}; fi" +prop_checkUnassignedReferences31 = verifyNotTree checkUnassignedReferences "X=1; if [[ -v foo[$X+42] ]]; then echo ${foo[$X+42]}; fi" +prop_checkUnassignedReferences32 = verifyNotTree checkUnassignedReferences "if [[ -v \"foo[1]\" ]]; then echo ${foo[@]}; fi" +prop_checkUnassignedReferences33 = verifyNotTree checkUnassignedReferences "f() { local -A foo; echo \"${foo[@]}\"; }" +prop_checkUnassignedReferences34 = verifyNotTree checkUnassignedReferences "declare -A foo; (( foo[bar] ))" +prop_checkUnassignedReferences35 = verifyNotTree checkUnassignedReferences "echo ${arr[foo-bar]:?fail}" +prop_checkUnassignedReferences36 = verifyNotTree checkUnassignedReferences "read -a foo -r <<<\"foo bar\"; echo \"$foo\"" +prop_checkUnassignedReferences37 = verifyNotTree checkUnassignedReferences "var=howdy; printf -v 'array[0]' %s \"$var\"; printf %s \"${array[0]}\";" +prop_checkUnassignedReferences38 = verifyTree (checkUnassignedReferences' True) "echo $VAR" +prop_checkUnassignedReferences39 = verifyNotTree checkUnassignedReferences "builtin export var=4; echo $var" +prop_checkUnassignedReferences40 = verifyNotTree checkUnassignedReferences ": ${foo=bar}" +prop_checkUnassignedReferences41 = verifyNotTree checkUnassignedReferences "mapfile -t files 123; echo \"${files[@]}\"" +prop_checkUnassignedReferences42 = verifyNotTree checkUnassignedReferences "mapfile files -t; echo \"${files[@]}\"" +prop_checkUnassignedReferences43 = verifyNotTree checkUnassignedReferences "mapfile --future files; echo \"${files[@]}\"" prop_checkUnassignedReferences_minusNPlain = verifyNotTree checkUnassignedReferences "if [ -n \"$x\" ]; then echo $x; fi" prop_checkUnassignedReferences_minusZPlain = verifyNotTree checkUnassignedReferences "if [ -z \"$x\" ]; then echo \"\"; fi" prop_checkUnassignedReferences_minusNBraced = verifyNotTree checkUnassignedReferences "if [ -n \"${x}\" ]; then echo $x; fi" @@ -2792,11 +2792,11 @@ prop_checkUnpassedInFunctions6 = verifyNotTree checkUnpassedInFunctions "foo() { prop_checkUnpassedInFunctions7 = verifyTree checkUnpassedInFunctions "foo() { echo $1; }; foo; foo;" prop_checkUnpassedInFunctions8 = verifyNotTree checkUnpassedInFunctions "foo() { echo $((1)); }; foo;" prop_checkUnpassedInFunctions9 = verifyNotTree checkUnpassedInFunctions "foo() { echo $(($b)); }; foo;" -prop_checkUnpassedInFunctions10= verifyNotTree checkUnpassedInFunctions "foo() { echo $!; }; foo;" -prop_checkUnpassedInFunctions11= verifyNotTree checkUnpassedInFunctions "foo() { bar() { echo $1; }; bar baz; }; foo;" -prop_checkUnpassedInFunctions12= verifyNotTree checkUnpassedInFunctions "foo() { echo ${!var*}; }; foo;" -prop_checkUnpassedInFunctions13= verifyNotTree checkUnpassedInFunctions "# shellcheck disable=SC2120\nfoo() { echo $1; }\nfoo\n" -prop_checkUnpassedInFunctions14= verifyTree checkUnpassedInFunctions "foo() { echo $#; }; foo" +prop_checkUnpassedInFunctions10 = verifyNotTree checkUnpassedInFunctions "foo() { echo $!; }; foo;" +prop_checkUnpassedInFunctions11 = verifyNotTree checkUnpassedInFunctions "foo() { bar() { echo $1; }; bar baz; }; foo;" +prop_checkUnpassedInFunctions12 = verifyNotTree checkUnpassedInFunctions "foo() { echo ${!var*}; }; foo;" +prop_checkUnpassedInFunctions13 = verifyNotTree checkUnpassedInFunctions "# shellcheck disable=SC2120\nfoo() { echo $1; }\nfoo\n" +prop_checkUnpassedInFunctions14 = verifyTree checkUnpassedInFunctions "foo() { echo $#; }; foo" checkUnpassedInFunctions params root = execWriter $ mapM_ warnForGroup referenceGroups where @@ -2971,12 +2971,12 @@ checkSuspiciousIFS params (T_Assignment _ _ "IFS" [] value) = checkSuspiciousIFS _ _ = return () -prop_checkGrepQ1= verify checkShouldUseGrepQ "[[ $(foo | grep bar) ]]" -prop_checkGrepQ2= verify checkShouldUseGrepQ "[ -z $(fgrep lol) ]" -prop_checkGrepQ3= verify checkShouldUseGrepQ "[ -n \"$(foo | zgrep lol)\" ]" -prop_checkGrepQ4= verifyNot checkShouldUseGrepQ "[ -z $(grep bar | cmd) ]" -prop_checkGrepQ5= verifyNot checkShouldUseGrepQ "rm $(ls | grep file)" -prop_checkGrepQ6= verifyNot checkShouldUseGrepQ "[[ -n $(pgrep foo) ]]" +prop_checkGrepQ1 = verify checkShouldUseGrepQ "[[ $(foo | grep bar) ]]" +prop_checkGrepQ2 = verify checkShouldUseGrepQ "[ -z $(fgrep lol) ]" +prop_checkGrepQ3 = verify checkShouldUseGrepQ "[ -n \"$(foo | zgrep lol)\" ]" +prop_checkGrepQ4 = verifyNot checkShouldUseGrepQ "[ -z $(grep bar | cmd) ]" +prop_checkGrepQ5 = verifyNot checkShouldUseGrepQ "rm $(ls | grep file)" +prop_checkGrepQ6 = verifyNot checkShouldUseGrepQ "[[ -n $(pgrep foo) ]]" checkShouldUseGrepQ params t = sequence_ $ case t of TC_Nullary id _ token -> check id True token diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index acbf967..4bae17e 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -216,18 +216,18 @@ checker spec params = getChecker $ commandChecks ++ optionals prop_checkTr1 = verify checkTr "tr [a-f] [A-F]" prop_checkTr2 = verify checkTr "tr 'a-z' 'A-Z'" -prop_checkTr2a= verify checkTr "tr '[a-z]' '[A-Z]'" +prop_checkTr2a = verify checkTr "tr '[a-z]' '[A-Z]'" prop_checkTr3 = verifyNot checkTr "tr -d '[:lower:]'" -prop_checkTr3a= verifyNot checkTr "tr -d '[:upper:]'" -prop_checkTr3b= verifyNot checkTr "tr -d '|/_[:upper:]'" +prop_checkTr3a = verifyNot checkTr "tr -d '[:upper:]'" +prop_checkTr3b = verifyNot checkTr "tr -d '|/_[:upper:]'" prop_checkTr4 = verifyNot checkTr "ls [a-z]" prop_checkTr5 = verify checkTr "tr foo bar" prop_checkTr6 = verify checkTr "tr 'hello' 'world'" prop_checkTr8 = verifyNot checkTr "tr aeiou _____" prop_checkTr9 = verifyNot checkTr "a-z n-za-m" -prop_checkTr10= verifyNot checkTr "tr --squeeze-repeats rl lr" -prop_checkTr11= verifyNot checkTr "tr abc '[d*]'" -prop_checkTr12= verifyNot checkTr "tr '[=e=]' 'e'" +prop_checkTr10 = verifyNot checkTr "tr --squeeze-repeats rl lr" +prop_checkTr11 = verifyNot checkTr "tr abc '[d*]'" +prop_checkTr12 = verifyNot checkTr "tr '[=e=]' 'e'" checkTr = CommandCheck (Basename "tr") (mapM_ f . arguments) where f w | isGlob w = -- The user will go [ab] -> '[ab]' -> 'ab'. Fixme? @@ -339,20 +339,20 @@ prop_checkGrepRe6 = verifyNot checkGrepRe "grep foo \\*.mp3" prop_checkGrepRe7 = verify checkGrepRe "grep *foo* file" prop_checkGrepRe8 = verify checkGrepRe "ls | grep foo*.jpg" prop_checkGrepRe9 = verifyNot checkGrepRe "grep '[0-9]*' file" -prop_checkGrepRe10= verifyNot checkGrepRe "grep '^aa*' file" -prop_checkGrepRe11= verifyNot checkGrepRe "grep --include=*.png foo" -prop_checkGrepRe12= verifyNot checkGrepRe "grep -F 'Foo*' file" -prop_checkGrepRe13= verifyNot checkGrepRe "grep -- -foo bar*" -prop_checkGrepRe14= verifyNot checkGrepRe "grep -e -foo bar*" -prop_checkGrepRe15= verifyNot checkGrepRe "grep --regex -foo bar*" -prop_checkGrepRe16= verifyNot checkGrepRe "grep --include 'Foo*' file" -prop_checkGrepRe17= verifyNot checkGrepRe "grep --exclude 'Foo*' file" -prop_checkGrepRe18= verifyNot checkGrepRe "grep --exclude-dir 'Foo*' file" -prop_checkGrepRe19= verify checkGrepRe "grep -- 'Foo*' file" -prop_checkGrepRe20= verifyNot checkGrepRe "grep --fixed-strings 'Foo*' file" -prop_checkGrepRe21= verifyNot checkGrepRe "grep -o 'x*' file" -prop_checkGrepRe22= verifyNot checkGrepRe "grep --only-matching 'x*' file" -prop_checkGrepRe23= verifyNot checkGrepRe "grep '.*' file" +prop_checkGrepRe10 = verifyNot checkGrepRe "grep '^aa*' file" +prop_checkGrepRe11 = verifyNot checkGrepRe "grep --include=*.png foo" +prop_checkGrepRe12 = verifyNot checkGrepRe "grep -F 'Foo*' file" +prop_checkGrepRe13 = verifyNot checkGrepRe "grep -- -foo bar*" +prop_checkGrepRe14 = verifyNot checkGrepRe "grep -e -foo bar*" +prop_checkGrepRe15 = verifyNot checkGrepRe "grep --regex -foo bar*" +prop_checkGrepRe16 = verifyNot checkGrepRe "grep --include 'Foo*' file" +prop_checkGrepRe17 = verifyNot checkGrepRe "grep --exclude 'Foo*' file" +prop_checkGrepRe18 = verifyNot checkGrepRe "grep --exclude-dir 'Foo*' file" +prop_checkGrepRe19 = verify checkGrepRe "grep -- 'Foo*' file" +prop_checkGrepRe20 = verifyNot checkGrepRe "grep --fixed-strings 'Foo*' file" +prop_checkGrepRe21 = verifyNot checkGrepRe "grep -o 'x*' file" +prop_checkGrepRe22 = verifyNot checkGrepRe "grep --only-matching 'x*' file" +prop_checkGrepRe23 = verifyNot checkGrepRe "grep '.*' file" checkGrepRe = CommandCheck (Basename "grep") check where check cmd = f cmd (arguments cmd) @@ -400,7 +400,7 @@ checkGrepRe = CommandCheck (Basename "grep") check where prop_checkTrapQuotes1 = verify checkTrapQuotes "trap \"echo $num\" INT" -prop_checkTrapQuotes1a= verify checkTrapQuotes "trap \"echo `ls`\" INT" +prop_checkTrapQuotes1a = verify checkTrapQuotes "trap \"echo `ls`\" INT" prop_checkTrapQuotes2 = verifyNot checkTrapQuotes "trap 'echo $num' INT" prop_checkTrapQuotes3 = verify checkTrapQuotes "trap \"echo $((1+num))\" EXIT DEBUG" checkTrapQuotes = CommandCheck (Exactly "trap") (f . arguments) where @@ -657,19 +657,19 @@ prop_checkPrintfVar6 = verify checkPrintfVar "printf foo bar baz" prop_checkPrintfVar7 = verify checkPrintfVar "printf -- foo bar baz" prop_checkPrintfVar8 = verifyNot checkPrintfVar "printf '%s %s %s' \"${var[@]}\"" prop_checkPrintfVar9 = verifyNot checkPrintfVar "printf '%s %s %s\\n' *.png" -prop_checkPrintfVar10= verifyNot checkPrintfVar "printf '%s %s %s' foo bar baz" -prop_checkPrintfVar11= verifyNot checkPrintfVar "printf '%(%s%s)T' -1" -prop_checkPrintfVar12= verify checkPrintfVar "printf '%s %s\\n' 1 2 3" -prop_checkPrintfVar13= verifyNot checkPrintfVar "printf '%s %s\\n' 1 2 3 4" -prop_checkPrintfVar14= verify checkPrintfVar "printf '%*s\\n' 1" -prop_checkPrintfVar15= verifyNot checkPrintfVar "printf '%*s\\n' 1 2" -prop_checkPrintfVar16= verifyNot checkPrintfVar "printf $'string'" -prop_checkPrintfVar17= verify checkPrintfVar "printf '%-*s\\n' 1" -prop_checkPrintfVar18= verifyNot checkPrintfVar "printf '%-*s\\n' 1 2" -prop_checkPrintfVar19= verifyNot checkPrintfVar "printf '%(%s)T'" -prop_checkPrintfVar20= verifyNot checkPrintfVar "printf '%d %(%s)T' 42" -prop_checkPrintfVar21= verify checkPrintfVar "printf '%d %(%s)T'" -prop_checkPrintfVar22= verify checkPrintfVar "printf '%s\n%s' foo" +prop_checkPrintfVar10 = verifyNot checkPrintfVar "printf '%s %s %s' foo bar baz" +prop_checkPrintfVar11 = verifyNot checkPrintfVar "printf '%(%s%s)T' -1" +prop_checkPrintfVar12 = verify checkPrintfVar "printf '%s %s\\n' 1 2 3" +prop_checkPrintfVar13 = verifyNot checkPrintfVar "printf '%s %s\\n' 1 2 3 4" +prop_checkPrintfVar14 = verify checkPrintfVar "printf '%*s\\n' 1" +prop_checkPrintfVar15 = verifyNot checkPrintfVar "printf '%*s\\n' 1 2" +prop_checkPrintfVar16 = verifyNot checkPrintfVar "printf $'string'" +prop_checkPrintfVar17 = verify checkPrintfVar "printf '%-*s\\n' 1" +prop_checkPrintfVar18 = verifyNot checkPrintfVar "printf '%-*s\\n' 1 2" +prop_checkPrintfVar19 = verifyNot checkPrintfVar "printf '%(%s)T'" +prop_checkPrintfVar20 = verifyNot checkPrintfVar "printf '%d %(%s)T' 42" +prop_checkPrintfVar21 = verify checkPrintfVar "printf '%d %(%s)T'" +prop_checkPrintfVar22 = verify checkPrintfVar "printf '%s\n%s' foo" checkPrintfVar = CommandCheck (Exactly "printf") (f . arguments) where f (doubledash:rest) | getLiteralString doubledash == Just "--" = f rest @@ -1069,10 +1069,10 @@ prop_checkCatastrophicRm4 = verify checkCatastrophicRm "rm -fr /home/$(whoami)/* prop_checkCatastrophicRm5 = verifyNot checkCatastrophicRm "rm -r /home/${USER:-thing}/*" prop_checkCatastrophicRm6 = verify checkCatastrophicRm "rm --recursive /etc/*$config*" prop_checkCatastrophicRm8 = verify checkCatastrophicRm "rm -rf /home" -prop_checkCatastrophicRm10= verifyNot checkCatastrophicRm "rm -r \"${DIR}\"/{.gitignore,.gitattributes,ci}" -prop_checkCatastrophicRm11= verify checkCatastrophicRm "rm -r /{bin,sbin}/$exec" -prop_checkCatastrophicRm12= verify checkCatastrophicRm "rm -r /{{usr,},{bin,sbin}}/$exec" -prop_checkCatastrophicRm13= verifyNot checkCatastrophicRm "rm -r /{{a,b},{c,d}}/$exec" +prop_checkCatastrophicRm10 = verifyNot checkCatastrophicRm "rm -r \"${DIR}\"/{.gitignore,.gitattributes,ci}" +prop_checkCatastrophicRm11 = verify checkCatastrophicRm "rm -r /{bin,sbin}/$exec" +prop_checkCatastrophicRm12 = verify checkCatastrophicRm "rm -r /{{usr,},{bin,sbin}}/$exec" +prop_checkCatastrophicRm13 = verifyNot checkCatastrophicRm "rm -r /{{a,b},{c,d}}/$exec" prop_checkCatastrophicRmA = verify checkCatastrophicRm "rm -rf /usr /lib/nvidia-current/xorg/xorg" prop_checkCatastrophicRmB = verify checkCatastrophicRm "rm -rf \"$STEAMROOT/\"*" checkCatastrophicRm = CommandCheck (Basename "rm") $ \t -> diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 9ad17f5..c12da2d 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -92,55 +92,55 @@ prop_checkBashisms6 = verify checkBashisms "[ \"$a\" == 42 ]" prop_checkBashisms7 = verify checkBashisms "echo ${var[1]}" prop_checkBashisms8 = verify checkBashisms "echo ${!var[@]}" prop_checkBashisms9 = verify checkBashisms "echo ${!var*}" -prop_checkBashisms10= verify checkBashisms "echo ${var:4:12}" -prop_checkBashisms11= verifyNot checkBashisms "echo ${var:-4}" -prop_checkBashisms12= verify checkBashisms "echo ${var//foo/bar}" -prop_checkBashisms13= verify checkBashisms "exec -c env" -prop_checkBashisms14= verify checkBashisms "echo -n \"Foo: \"" -prop_checkBashisms15= verify checkBashisms "let n++" -prop_checkBashisms16= verify checkBashisms "echo $RANDOM" -prop_checkBashisms17= verify checkBashisms "echo $((RANDOM%6+1))" -prop_checkBashisms18= verify checkBashisms "foo &> /dev/null" -prop_checkBashisms19= verify checkBashisms "foo > file*.txt" -prop_checkBashisms20= verify checkBashisms "read -ra foo" -prop_checkBashisms21= verify checkBashisms "[ -a foo ]" -prop_checkBashisms22= verifyNot checkBashisms "[ foo -a bar ]" -prop_checkBashisms23= verify checkBashisms "trap mything ERR INT" -prop_checkBashisms24= verifyNot checkBashisms "trap mything INT TERM" -prop_checkBashisms25= verify checkBashisms "cat < /dev/tcp/host/123" -prop_checkBashisms26= verify checkBashisms "trap mything ERR SIGTERM" -prop_checkBashisms27= verify checkBashisms "echo *[^0-9]*" -prop_checkBashisms28= verify checkBashisms "exec {n}>&2" -prop_checkBashisms29= verify checkBashisms "echo ${!var}" -prop_checkBashisms30= verify checkBashisms "printf -v '%s' \"$1\"" -prop_checkBashisms31= verify checkBashisms "printf '%q' \"$1\"" -prop_checkBashisms32= verifyNot checkBashisms "#!/bin/dash\n[ foo -nt bar ]" -prop_checkBashisms33= verify checkBashisms "#!/bin/sh\necho -n foo" -prop_checkBashisms34= verifyNot checkBashisms "#!/bin/dash\necho -n foo" -prop_checkBashisms35= verifyNot checkBashisms "#!/bin/dash\nlocal foo" -prop_checkBashisms36= verifyNot checkBashisms "#!/bin/dash\nread -p foo -r bar" -prop_checkBashisms37= verifyNot checkBashisms "HOSTNAME=foo; echo $HOSTNAME" -prop_checkBashisms38= verify checkBashisms "RANDOM=9; echo $RANDOM" -prop_checkBashisms39= verify checkBashisms "foo-bar() { true; }" -prop_checkBashisms40= verify checkBashisms "echo $(/dev/null" -prop_checkBashisms48= verifyNot checkBashisms "#!/bin/sh\necho $LINENO" -prop_checkBashisms49= verify checkBashisms "#!/bin/dash\necho $MACHTYPE" -prop_checkBashisms50= verify checkBashisms "#!/bin/sh\ncmd >& file" -prop_checkBashisms51= verifyNot checkBashisms "#!/bin/sh\ncmd 2>&1" -prop_checkBashisms52= verifyNot checkBashisms "#!/bin/sh\ncmd >&2" -prop_checkBashisms53= verifyNot checkBashisms "#!/bin/sh\nprintf -- -f\n" -prop_checkBashisms54= verify checkBashisms "#!/bin/sh\nfoo+=bar" -prop_checkBashisms55= verify checkBashisms "#!/bin/sh\necho ${@%foo}" -prop_checkBashisms56= verifyNot checkBashisms "#!/bin/sh\necho ${##}" -prop_checkBashisms57= verifyNot checkBashisms "#!/bin/dash\nulimit -c 0" -prop_checkBashisms58= verify checkBashisms "#!/bin/sh\nulimit -c 0" +prop_checkBashisms10 = verify checkBashisms "echo ${var:4:12}" +prop_checkBashisms11 = verifyNot checkBashisms "echo ${var:-4}" +prop_checkBashisms12 = verify checkBashisms "echo ${var//foo/bar}" +prop_checkBashisms13 = verify checkBashisms "exec -c env" +prop_checkBashisms14 = verify checkBashisms "echo -n \"Foo: \"" +prop_checkBashisms15 = verify checkBashisms "let n++" +prop_checkBashisms16 = verify checkBashisms "echo $RANDOM" +prop_checkBashisms17 = verify checkBashisms "echo $((RANDOM%6+1))" +prop_checkBashisms18 = verify checkBashisms "foo &> /dev/null" +prop_checkBashisms19 = verify checkBashisms "foo > file*.txt" +prop_checkBashisms20 = verify checkBashisms "read -ra foo" +prop_checkBashisms21 = verify checkBashisms "[ -a foo ]" +prop_checkBashisms22 = verifyNot checkBashisms "[ foo -a bar ]" +prop_checkBashisms23 = verify checkBashisms "trap mything ERR INT" +prop_checkBashisms24 = verifyNot checkBashisms "trap mything INT TERM" +prop_checkBashisms25 = verify checkBashisms "cat < /dev/tcp/host/123" +prop_checkBashisms26 = verify checkBashisms "trap mything ERR SIGTERM" +prop_checkBashisms27 = verify checkBashisms "echo *[^0-9]*" +prop_checkBashisms28 = verify checkBashisms "exec {n}>&2" +prop_checkBashisms29 = verify checkBashisms "echo ${!var}" +prop_checkBashisms30 = verify checkBashisms "printf -v '%s' \"$1\"" +prop_checkBashisms31 = verify checkBashisms "printf '%q' \"$1\"" +prop_checkBashisms32 = verifyNot checkBashisms "#!/bin/dash\n[ foo -nt bar ]" +prop_checkBashisms33 = verify checkBashisms "#!/bin/sh\necho -n foo" +prop_checkBashisms34 = verifyNot checkBashisms "#!/bin/dash\necho -n foo" +prop_checkBashisms35 = verifyNot checkBashisms "#!/bin/dash\nlocal foo" +prop_checkBashisms36 = verifyNot checkBashisms "#!/bin/dash\nread -p foo -r bar" +prop_checkBashisms37 = verifyNot checkBashisms "HOSTNAME=foo; echo $HOSTNAME" +prop_checkBashisms38 = verify checkBashisms "RANDOM=9; echo $RANDOM" +prop_checkBashisms39 = verify checkBashisms "foo-bar() { true; }" +prop_checkBashisms40 = verify checkBashisms "echo $(/dev/null" +prop_checkBashisms48 = verifyNot checkBashisms "#!/bin/sh\necho $LINENO" +prop_checkBashisms49 = verify checkBashisms "#!/bin/dash\necho $MACHTYPE" +prop_checkBashisms50 = verify checkBashisms "#!/bin/sh\ncmd >& file" +prop_checkBashisms51 = verifyNot checkBashisms "#!/bin/sh\ncmd 2>&1" +prop_checkBashisms52 = verifyNot checkBashisms "#!/bin/sh\ncmd >&2" +prop_checkBashisms53 = verifyNot checkBashisms "#!/bin/sh\nprintf -- -f\n" +prop_checkBashisms54 = verify checkBashisms "#!/bin/sh\nfoo+=bar" +prop_checkBashisms55 = verify checkBashisms "#!/bin/sh\necho ${@%foo}" +prop_checkBashisms56 = verifyNot checkBashisms "#!/bin/sh\necho ${##}" +prop_checkBashisms57 = verifyNot checkBashisms "#!/bin/dash\nulimit -c 0" +prop_checkBashisms58 = verify checkBashisms "#!/bin/sh\nulimit -c 0" prop_checkBashisms59 = verify checkBashisms "#!/bin/sh\njobs -s" prop_checkBashisms60 = verifyNot checkBashisms "#!/bin/sh\njobs -p" prop_checkBashisms61 = verifyNot checkBashisms "#!/bin/sh\njobs -lp" @@ -441,9 +441,9 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do _ -> False prop_checkEchoSed1 = verify checkEchoSed "FOO=$(echo \"$cow\" | sed 's/foo/bar/g')" -prop_checkEchoSed1b= verify checkEchoSed "FOO=$(sed 's/foo/bar/g' <<< \"$cow\")" +prop_checkEchoSed1b = verify checkEchoSed "FOO=$(sed 's/foo/bar/g' <<< \"$cow\")" prop_checkEchoSed2 = verify checkEchoSed "rm $(echo $cow | sed -e 's,foo,bar,')" -prop_checkEchoSed2b= verify checkEchoSed "rm $(sed -e 's,foo,bar,' <<< $cow)" +prop_checkEchoSed2b = verify checkEchoSed "rm $(sed -e 's,foo,bar,' <<< $cow)" checkEchoSed = ForShell [Bash, Ksh] f where f (T_Redirecting id lefts r) = @@ -529,11 +529,11 @@ checkMultiDimensionalArrays = ForShell [Bash] f isMultiDim l = getBracedModifier (concat $ oversimplify l) `matches` re prop_checkPS11 = verify checkPS1Assignments "PS1='\\033[1;35m\\$ '" -prop_checkPS11a= verify checkPS1Assignments "export PS1='\\033[1;35m\\$ '" +prop_checkPS11a = verify checkPS1Assignments "export PS1='\\033[1;35m\\$ '" prop_checkPSf2 = verify checkPS1Assignments "PS1='\\h \\e[0m\\$ '" prop_checkPS13 = verify checkPS1Assignments "PS1=$'\\x1b[c '" prop_checkPS14 = verify checkPS1Assignments "PS1=$'\\e[3m; '" -prop_checkPS14a= verify checkPS1Assignments "export PS1=$'\\e[3m; '" +prop_checkPS14a = verify checkPS1Assignments "export PS1=$'\\e[3m; '" prop_checkPS15 = verifyNot checkPS1Assignments "PS1='\\[\\033[1;35m\\]\\$ '" prop_checkPS16 = verifyNot checkPS1Assignments "PS1='\\[\\e1m\\e[1m\\]\\$ '" prop_checkPS17 = verifyNot checkPS1Assignments "PS1='e033x1B'" diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 9f9241c..e6a2999 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -719,20 +719,20 @@ prop_a6 = isOk readArithmeticContents " 1 | 2 ||3|4" prop_a7 = isOk readArithmeticContents "3*2**10" prop_a8 = isOk readArithmeticContents "3" prop_a9 = isOk readArithmeticContents "a^!-b" -prop_a10= isOk readArithmeticContents "! $?" -prop_a11= isOk readArithmeticContents "10#08 * 16#f" -prop_a12= isOk readArithmeticContents "\"$((3+2))\" + '37'" -prop_a13= isOk readArithmeticContents "foo[9*y+x]++" -prop_a14= isOk readArithmeticContents "1+`echo 2`" -prop_a15= isOk readArithmeticContents "foo[`echo foo | sed s/foo/4/g` * 3] + 4" -prop_a16= isOk readArithmeticContents "$foo$bar" -prop_a17= isOk readArithmeticContents "i<(0+(1+1))" -prop_a18= isOk readArithmeticContents "a?b:c" -prop_a19= isOk readArithmeticContents "\\\n3 +\\\n 2" -prop_a20= isOk readArithmeticContents "a ? b ? c : d : e" -prop_a21= isOk readArithmeticContents "a ? b : c ? d : e" -prop_a22= isOk readArithmeticContents "!!a" -prop_a23= isOk readArithmeticContents "~0" +prop_a10 = isOk readArithmeticContents "! $?" +prop_a11 = isOk readArithmeticContents "10#08 * 16#f" +prop_a12 = isOk readArithmeticContents "\"$((3+2))\" + '37'" +prop_a13 = isOk readArithmeticContents "foo[9*y+x]++" +prop_a14 = isOk readArithmeticContents "1+`echo 2`" +prop_a15 = isOk readArithmeticContents "foo[`echo foo | sed s/foo/4/g` * 3] + 4" +prop_a16 = isOk readArithmeticContents "$foo$bar" +prop_a17 = isOk readArithmeticContents "i<(0+(1+1))" +prop_a18 = isOk readArithmeticContents "a?b:c" +prop_a19 = isOk readArithmeticContents "\\\n3 +\\\n 2" +prop_a20 = isOk readArithmeticContents "a ? b ? c : d : e" +prop_a21 = isOk readArithmeticContents "a ? b : c ? d : e" +prop_a22 = isOk readArithmeticContents "!!a" +prop_a23 = isOk readArithmeticContents "~0" readArithmeticContents :: Monad m => SCParser m Token readArithmeticContents = readSequence @@ -925,8 +925,8 @@ prop_readCondition7 = isOk readCondition "[[ ${line} =~ ^[[:space:]]*# ]]" prop_readCondition8 = isOk readCondition "[[ $l =~ ogg|flac ]]" prop_readCondition9 = isOk readCondition "[ foo -a -f bar ]" prop_readCondition10 = isOk readCondition "[[\na == b\n||\nc == d ]]" -prop_readCondition10a= isOk readCondition "[[\na == b ||\nc == d ]]" -prop_readCondition10b= isOk readCondition "[[ a == b\n||\nc == d ]]" +prop_readCondition10a = isOk readCondition "[[\na == b ||\nc == d ]]" +prop_readCondition10b = isOk readCondition "[[ a == b\n||\nc == d ]]" prop_readCondition11 = isOk readCondition "[[ a == b ||\n c == d ]]" prop_readCondition12 = isWarning readCondition "[ a == b \n -o c == d ]" prop_readCondition13 = isOk readCondition "[[ foo =~ ^fo{1,3}$ ]]" @@ -1701,9 +1701,9 @@ readDollarBraced = called "parameter expansion" $ do id <- endSpan start return $ T_DollarBraced id True word -prop_readDollarExpansion1= isOk readDollarExpansion "$(echo foo; ls\n)" -prop_readDollarExpansion2= isOk readDollarExpansion "$( )" -prop_readDollarExpansion3= isOk readDollarExpansion "$( command \n#comment \n)" +prop_readDollarExpansion1 = isOk readDollarExpansion "$(echo foo; ls\n)" +prop_readDollarExpansion2 = isOk readDollarExpansion "$( )" +prop_readDollarExpansion3 = isOk readDollarExpansion "$( command \n#comment \n)" readDollarExpansion = called "command expansion" $ do start <- startSpan try (string "$(") @@ -1795,17 +1795,17 @@ prop_readHereDoc6 = isOk readScript "cat << foo\\ bar\ncow\nfoo bar" prop_readHereDoc7 = isOk readScript "cat << foo\n\\$(f ())\nfoo" prop_readHereDoc8 = isOk readScript "cat <>bar\netc\nfoo" prop_readHereDoc9 = isOk readScript "if true; then cat << foo; fi\nbar\nfoo\n" -prop_readHereDoc10= isOk readScript "if true; then cat << foo << bar; fi\nfoo\nbar\n" -prop_readHereDoc11= isOk readScript "cat << foo $(\nfoo\n)lol\nfoo\n" -prop_readHereDoc12= isOk readScript "cat << foo|cat\nbar\nfoo" -prop_readHereDoc13= isOk readScript "cat <<'#!'\nHello World\n#!\necho Done" -prop_readHereDoc14= isWarning readScript "cat << foo\nbar\nfoo \n" -prop_readHereDoc15= isWarning readScript "cat < Date: Fri, 22 Jul 2022 17:06:24 -0700 Subject: [PATCH 552/763] Omit SC3021 about `>& file` unless definitely non-numeric (fixes #2520) --- src/ShellCheck/Checks/ShellSupport.hs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index c12da2d..30a19b9 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -135,6 +135,8 @@ prop_checkBashisms49 = verify checkBashisms "#!/bin/dash\necho $MACHTYPE" prop_checkBashisms50 = verify checkBashisms "#!/bin/sh\ncmd >& file" prop_checkBashisms51 = verifyNot checkBashisms "#!/bin/sh\ncmd 2>&1" prop_checkBashisms52 = verifyNot checkBashisms "#!/bin/sh\ncmd >&2" +prop_checkBashisms52b = verifyNot checkBashisms "#!/bin/sh\ncmd >& $var" +prop_checkBashisms52c = verify checkBashisms "#!/bin/sh\ncmd >& $dir/$var" prop_checkBashisms53 = verifyNot checkBashisms "#!/bin/sh\nprintf -- -f\n" prop_checkBashisms54 = verify checkBashisms "#!/bin/sh\nfoo+=bar" prop_checkBashisms55 = verify checkBashisms "#!/bin/sh\necho ${@%foo}" @@ -225,7 +227,8 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do warnMsg id 3018 $ filter (/= '|') op ++ " is" bashism (TA_Binary id "**" _ _) = warnMsg id 3019 "exponentials are" bashism (T_FdRedirect id "&" (T_IoFile _ (T_Greater _) _)) = warnMsg id 3020 "&> is" - bashism (T_FdRedirect id "" (T_IoFile _ (T_GREATAND _) _)) = warnMsg id 3021 ">& is" + bashism (T_FdRedirect id "" (T_IoFile _ (T_GREATAND _) file)) = + unless (all isDigit $ onlyLiteralString file) $ warnMsg id 3021 ">& filename (as opposed to >& fd) is" bashism (T_FdRedirect id ('{':_) _) = warnMsg id 3022 "named file descriptors are" bashism (T_FdRedirect id num _) | all isDigit num && length num > 1 = warnMsg id 3023 "FDs outside 0-9 are" From b261ec24f9ebcb911e7ff4264e5ea7d36cc93f59 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Fri, 22 Jul 2022 20:16:01 -0700 Subject: [PATCH 553/763] Include exit codes in DFA (ref #2541) --- src/ShellCheck/CFG.hs | 8 ++- src/ShellCheck/CFGAnalysis.hs | 92 +++++++++++++++++++++++++++-------- 2 files changed, 77 insertions(+), 23 deletions(-) diff --git a/src/ShellCheck/CFG.hs b/src/ShellCheck/CFG.hs index 4906d80..1085d8f 100644 --- a/src/ShellCheck/CFG.hs +++ b/src/ShellCheck/CFG.hs @@ -651,7 +651,10 @@ build t = do pg <- wordToExactPseudoGlob c return $ pg `pseudoGlobIsSuperSetof` [PGMany] - T_Condition _ _ op -> build op + T_Condition id _ op -> do + cond <- build op + status <- newNodeRange $ CFSetExitCode id + linkRange cond status T_CoProc id maybeName t -> do let name = fromMaybe "COPROC" maybeName @@ -798,7 +801,8 @@ build t = do start <- newStructuralNode hasLastpipe <- reader $ cfLastpipe . cfParameters (leading, last) <- buildPipe hasLastpipe cmds - end <- newStructuralNode + -- Ideally we'd let this exit code be that of the last command in the pipeline but ok + end <- newNodeRange $ CFSetExitCode id mapM_ (linkRange start) leading mapM_ (\c -> linkRangeAs CFEFalseFlow c end) leading diff --git a/src/ShellCheck/CFGAnalysis.hs b/src/ShellCheck/CFGAnalysis.hs index daade43..893c34a 100644 --- a/src/ShellCheck/CFGAnalysis.hs +++ b/src/ShellCheck/CFGAnalysis.hs @@ -104,11 +104,29 @@ data CFGAnalysis = CFGAnalysis { -- The program state we expose externally data ProgramState = ProgramState { --- internalState :: InternalState, -- For debugging + -- internalState :: InternalState, -- For debugging variablesInScope :: M.Map String VariableState, + exitCodes :: S.Set Id, stateIsReachable :: Bool } deriving (Show, Eq, Generic, NFData) +internalToExternal :: InternalState -> ProgramState +internalToExternal s = + ProgramState { + -- Censor the literal value to avoid introducing dependencies on it. It's just for debugging. + variablesInScope = M.map censor flatVars, + -- internalState = s, -- For debugging + exitCodes = fromMaybe S.empty $ sExitCodes s, + stateIsReachable = fromMaybe True $ sIsReachable s + } + where + censor s = s { + variableValue = (variableValue s) { + literalValue = Nothing + } + } + flatVars = M.unionsWith (\_ last -> last) $ map mapStorage [sGlobalValues s, sLocalValues s, sPrefixValues s] + -- Conveniently get the state before a token id getIncomingState :: CFGAnalysis -> Id -> Maybe ProgramState getIncomingState analysis id = do @@ -130,6 +148,7 @@ data InternalState = InternalState { sLocalValues :: VersionedMap String VariableState, sPrefixValues :: VersionedMap String VariableState, sFunctionTargets :: VersionedMap String FunctionValue, + sExitCodes :: Maybe (S.Set Id), sIsReachable :: Maybe Bool } deriving (Show, Generic, NFData) @@ -139,6 +158,7 @@ newInternalState = InternalState { sLocalValues = vmEmpty, sPrefixValues = vmEmpty, sFunctionTargets = vmEmpty, + sExitCodes = Nothing, sIsReachable = Nothing } @@ -196,31 +216,25 @@ removeProperties props state = state { variableProperties = S.map (\s -> S.difference s props) $ variableProperties state } -internalToExternal :: InternalState -> ProgramState -internalToExternal s = - ProgramState { - -- Censor the literal value to avoid introducing dependencies on it. It's just for debugging. - variablesInScope = M.map censor flatVars, - -- internalState = s, -- For debugging - stateIsReachable = fromMaybe True $ sIsReachable s - } - where - censor s = s { - variableValue = (variableValue s) { - literalValue = Nothing - } - } - flatVars = M.unionsWith (\_ last -> last) $ map mapStorage [sGlobalValues s, sLocalValues s, sPrefixValues s] +setExitCode id = setExitCodes (S.singleton id) +setExitCodes set state = modified state { + sExitCodes = Just $ set +} -- Dependencies on values, e.g. "if there is a global variable named 'foo' without spaces" -- This is used to see if the DFA of a function would result in the same state, so anything -- that affects DFA must be tracked. data StateDependency = + -- Complete variable state DepState Scope String VariableState + -- Only variable properties (we need properties but not values for x=1) | DepProperties Scope String VariableProperties + -- Function definition | DepFunction String (S.Set FunctionDefinition) -- Whether invoking the node would result in recursion (i.e., is the function on the stack?) | DepIsRecursive Node Bool + -- The set of commands that could have provided the exit code $? + | DepExitCodes (S.Set Id) deriving (Show, Eq, Ord, Generic, NFData) -- A function definition, or lack thereof @@ -242,6 +256,7 @@ depsToState set = foldl insert newInternalState $ S.toList set -- State includes properties and more, so don't overwrite a state with properties DepProperties scope name props -> insertIn False scope name unknownVariableState { variableProperties = props } state DepIsRecursive _ _ -> state + DepExitCodes s -> setExitCodes s state insertIn overwrite scope name val state = let @@ -400,6 +415,7 @@ patchState base diff = sLocalValues = vmPatch (sLocalValues base) (sLocalValues diff), sPrefixValues = vmPatch (sPrefixValues base) (sPrefixValues diff), sFunctionTargets = vmPatch (sFunctionTargets base) (sFunctionTargets diff), + sExitCodes = sExitCodes diff `mplus` sExitCodes base, sIsReachable = sIsReachable diff `mplus` sIsReachable base } @@ -444,12 +460,14 @@ mergeState ctx a b = do locals <- mergeMaps ctx mergeVariableState readVariable (sLocalValues a) (sLocalValues b) prefix <- mergeMaps ctx mergeVariableState readVariable (sPrefixValues a) (sPrefixValues b) funcs <- mergeMaps ctx S.union readFunction (sFunctionTargets a) (sFunctionTargets b) + exitCodes <- mergeMaybes ctx S.union readExitCodes (sExitCodes a) (sExitCodes b) return $ InternalState { sVersion = -1, sGlobalValues = globals, sLocalValues = locals, sPrefixValues = prefix, sFunctionTargets = funcs, + sExitCodes = exitCodes, sIsReachable = liftM2 (&&) (sIsReachable a) (sIsReachable b) } @@ -493,6 +511,18 @@ mergeMaps ctx merger reader a b = nv1 <- reader ctx k2 f ((k2, merger nv1 v2):l) l1 rest2 +-- Merge two Maybes, like mergeMaps for a single element +mergeMaybes ctx merger reader a b = + case (a, b) of + (Nothing, Nothing) -> return Nothing + (Just v1, Nothing) -> single v1 + (Nothing, Just v2) -> single v2 + (Just v1, Just v2) -> return $ Just $ merger v1 v2 + where + single val = do + result <- merger val <$> reader ctx + return $ Just result + vmFromMap ctx map = return $ VersionedMap { mapVersion = -1, mapStorage = map @@ -708,6 +738,12 @@ readFunction ctx name = lookupStack get dep def ctx name writeFunction ctx name val = do modifySTRef (cOutput ctx) $ insertFunction name $ S.singleton val +readExitCodes ctx = lookupStack get dep def ctx () + where + get s () = sExitCodes s + def = S.empty + dep () v = DepExitCodes v + -- Look up each state on the stack until a value is found (or the default is used), -- then add this value as a StateDependency. lookupStack' :: forall s k v. @@ -872,13 +908,13 @@ transfer ctx label = CFExecuteCommand cmd -> transferCommand ctx cmd CFExecuteSubshell reason entry exit -> transferSubshell ctx reason entry exit CFApplyEffects effects -> mapM_ (\(IdTagged _ f) -> transferEffect ctx f) effects + CFSetExitCode id -> transferExitCode ctx id CFUnresolvedExit -> patchOutputM ctx unreachableState CFUnreachable -> patchOutputM ctx unreachableState -- TODO CFSetBackgroundPid _ -> return () - CFSetExitCode _ -> return () CFDropPrefixAssignments {} -> modifySTRef (cOutput ctx) $ \c -> modified c { sPrefixValues = vmEmpty } -- _ -> error $ "Unknown " ++ show label @@ -891,8 +927,11 @@ transferSubshell ctx reason entry exit = do let cout = cOutput ctx initial <- readSTRef cout runCached ctx entry (f entry exit) + res <- readSTRef cout -- Clear subshell changes. TODO: track this to warn about modifications. - writeSTRef cout initial + writeSTRef cout $ initial { + sExitCodes = sExitCodes res + } where f entry exit ctx = do (states, frame) <- withNewStackFrame ctx entry False (flip dataflow $ entry) @@ -947,6 +986,8 @@ transferFunctionValue ctx funcVal = registerFlowResult ctx entry states deps return (deps, res) +transferExitCode ctx id = do + modifySTRef (cOutput ctx) $ setExitCode id -- Register/save the result of a dataflow of a function. -- At the end, all the different values from different flows are merged together. @@ -1001,8 +1042,10 @@ getCache ctx node = do -- Transfer a single CFEffect to the output state. transferEffect ctx effect = case effect of - CFReadVariable name -> do - void $ readVariable ctx name + CFReadVariable name -> + case name of + "?" -> void $ readExitCodes ctx + _ -> void $ readVariable ctx name CFWriteVariable name value -> do val <- cfValueToVariableValue ctx value updateVariableValue ctx name val @@ -1235,7 +1278,14 @@ analyzeControlFlow params t = -- (it's probably not actually dead, just used by a script that sources ours) let declaredFunctions = getFunctionTargets exitState let uninvoked = M.difference declaredFunctions invokedNodes - analyzeStragglers ctx exitState uninvoked + + let stragglerInput = + exitState { + -- We don't want `die() { exit $?; }; echo "Sourced"` to assume $? is always echo + sExitCodes = Nothing + } + + analyzeStragglers ctx stragglerInput uninvoked -- Now round up all the states from all data flows -- (FIXME: this excludes functions that were defined in straggling functions) From f7857028f7a019b47c8a5294dd8d4182e5bed864 Mon Sep 17 00:00:00 2001 From: ygeyzel Date: Sat, 23 Jul 2022 19:24:16 +0300 Subject: [PATCH 554/763] Add escape characters to SC2028: \a, \b, \e, \f, \v, \\, \', \OOO, \xHH --- src/ShellCheck/Checks/Commands.hs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 4bae17e..67c3c48 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -481,9 +481,16 @@ prop_checkUnusedEchoEscapes2 = verifyNot checkUnusedEchoEscapes "echo -e 'foi\\n prop_checkUnusedEchoEscapes3 = verify checkUnusedEchoEscapes "echo \"n:\\t42\"" prop_checkUnusedEchoEscapes4 = verifyNot checkUnusedEchoEscapes "echo lol" prop_checkUnusedEchoEscapes5 = verifyNot checkUnusedEchoEscapes "echo -n -e '\n'" +prop_checkUnusedEchoEscapes6 = verify checkUnusedEchoEscapes "echo '\\506'" +prop_checkUnusedEchoEscapes7 = verify checkUnusedEchoEscapes "echo '\\5a'" +prop_checkUnusedEchoEscapes8 = verifyNot checkUnusedEchoEscapes "echo '\\8a'" +prop_checkUnusedEchoEscapes9 = verifyNot checkUnusedEchoEscapes "echo '\\d5a'" +prop_checkUnusedEchoEscapes10 = verify checkUnusedEchoEscapes "echo '\\x4a'" +prop_checkUnusedEchoEscapes11 = verify checkUnusedEchoEscapes "echo '\\xat'" +prop_checkUnusedEchoEscapes12 = verifyNot checkUnusedEchoEscapes "echo '\\xth'" checkUnusedEchoEscapes = CommandCheck (Basename "echo") f where - hasEscapes = mkRegex "\\\\[rnt]" + hasEscapes = mkRegex "\\\\([rntabefv\\']|[0-7]{1,3}|x([0-9]|[A-F]|[a-f]){1,2})" f cmd = whenShell [Sh, Bash, Ksh] $ unless (cmd `hasFlag` "e") $ From 5cf6e01ce902421065d8eee24c7a4a10d842efa9 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 23 Jul 2022 09:38:58 -0700 Subject: [PATCH 555/763] Warn when $? refers to echo or condition (ref #2541) --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 37 +++++++++++++++++++++++++++++++++++ src/ShellCheck/AnalyzerLib.hs | 3 +++ src/ShellCheck/CFGAnalysis.hs | 1 + 4 files changed, 42 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index dce3e27..5ff01c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ - SC2316: Warn about 'local readonly foo' and similar (thanks, patrickxia!) - SC2317: Warn about unreachable commands - SC2318: Warn about backreferences in 'declare x=1 y=$x' +- SC2319/SC2320: Warn when $? refers to echo/printf/[ ]/[[ ]]/test ### Fixed - SC2086: Now uses DFA to make more accurate predictions about values diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 1429e1b..07bf25b 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -205,6 +205,7 @@ nodeChecks = [ ,checkBatsTestDoesNotUseNegation ,checkCommandIsUnreachable ,checkSpacefulnessCfg + ,checkOverwrittenExitCode ] optionalChecks = map fst optionalTreeChecks @@ -4876,5 +4877,41 @@ checkCommandIsUnreachable params t = _ -> return () where id = getId t + +prop_checkOverwrittenExitCode1 = verify checkOverwrittenExitCode "x; [ $? -eq 1 ] || [ $? -eq 2 ]" +prop_checkOverwrittenExitCode2 = verifyNot checkOverwrittenExitCode "x; [ $? -eq 1 ]" +prop_checkOverwrittenExitCode3 = verify checkOverwrittenExitCode "x; echo \"Exit is $?\"; [ $? -eq 0 ]" +prop_checkOverwrittenExitCode4 = verifyNot checkOverwrittenExitCode "x; [ $? -eq 0 ]" +checkOverwrittenExitCode params t = + case t of + T_DollarBraced id _ val | getLiteralString val == Just "?" -> check id + _ -> return () + where + check id = sequence_ $ do + state <- CF.getIncomingState (cfgAnalysis params) id + let exitCodeIds = CF.exitCodes state + guard . not $ S.null exitCodeIds + + let idToToken = idMap params + exitCodeTokens <- sequence $ map (\k -> Map.lookup k idToToken) $ S.toList exitCodeIds + return $ do + when (all isCondition exitCodeTokens) $ + warn id 2319 "This $? refers to a condition, not a command. Assign to a variable to avoid it being overwritten." + when (all isPrinting exitCodeTokens) $ + warn id 2320 "This $? refers to echo/printf, not a previous command. Assign to variable to avoid it being overwritten." + + isCondition t = + case t of + T_Condition {} -> True + T_SimpleCommand {} -> getCommandName t == Just "test" + _ -> False + + isPrinting t = + case getCommandBasename t of + Just "echo" -> True + Just "printf" -> True + _ -> False + + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index e998f2c..88da89e 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -89,6 +89,8 @@ data Parameters = Parameters { hasPipefail :: Bool, -- A linear (bad) analysis of data flow variableFlow :: [StackData], + -- A map from Id to Token + idMap :: Map.Map Id Token, -- A map from Id to parent Token parentMap :: Map.Map Id Token, -- The shell type, such as Bash or Ksh @@ -218,6 +220,7 @@ makeParameters spec = params Sh -> True Ksh -> containsPipefail root, shellTypeSpecified = isJust (asShellType spec) || isJust (asFallbackShell spec), + idMap = getTokenMap root, parentMap = getParentTree root, variableFlow = getVariableFlow params root, tokenPositions = asTokenPositions spec, diff --git a/src/ShellCheck/CFGAnalysis.hs b/src/ShellCheck/CFGAnalysis.hs index 893c34a..bb90860 100644 --- a/src/ShellCheck/CFGAnalysis.hs +++ b/src/ShellCheck/CFGAnalysis.hs @@ -804,6 +804,7 @@ fulfillsDependency ctx entry dep = -- it won't be found by the normal check. DepIsRecursive node val | node == entry -> return True DepIsRecursive node val -> return $ val == any (\f -> entryPoint f == node) (cStack ctx) + DepExitCodes val -> (== val) <$> peekStack (\s k -> sExitCodes s) S.empty ctx () -- _ -> error $ "Unknown dep " ++ show dep where peek scope = peekStack getVariableWithScope $ if scope == GlobalScope then (unknownVariableState, GlobalScope) else (unsetVariableState, LocalScope) From ea4e0091c7b403ace135eddf1e508ff2fc0f783f Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 23 Jul 2022 15:38:42 -0700 Subject: [PATCH 556/763] Additionally pluralize 'arguments' in SC2183 --- src/ShellCheck/Checks/Commands.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 8bca2ba..e97ecd6 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -706,8 +706,8 @@ checkPrintfVar = CommandCheck (Exactly "printf") (f . arguments) where return () -- Great: a suitable number of arguments | otherwise -> warn (getId format) 2183 $ - "This format string has " ++ show formatCount ++ " " ++ (pluraliseIfMany "variable" formatCount) ++ - ", but is passed " ++ show argCount ++ " arguments." + "This format string has " ++ show formatCount ++ " " ++ pluraliseIfMany "variable" formatCount ++ + ", but is passed " ++ show argCount ++ pluraliseIfMany " argument" argCount ++ "." unless ('%' `elem` concat (oversimplify format) || isLiteral format) $ info (getId format) 2059 From 30bb0e0093172b93fecb53dca9d5d657962d8c0a Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 23 Jul 2022 20:10:58 -0700 Subject: [PATCH 557/763] SC2321: Warn about redundant $(()) in arr[$((i))]=x (ref: #1666) --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ff01c3..669579f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - SC2317: Warn about unreachable commands - SC2318: Warn about backreferences in 'declare x=1 y=$x' - SC2319/SC2320: Warn when $? refers to echo/printf/[ ]/[[ ]]/test +- SC2321: Suggest removing $((..)) in array[$((idx))]=val ### Fixed - SC2086: Now uses DFA to make more accurate predictions about values diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 07bf25b..e92f3ff 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -206,6 +206,7 @@ nodeChecks = [ ,checkCommandIsUnreachable ,checkSpacefulnessCfg ,checkOverwrittenExitCode + ,checkUnnecessaryArithmeticExpansionIndex ] optionalChecks = map fst optionalTreeChecks @@ -4913,5 +4914,22 @@ checkOverwrittenExitCode params t = _ -> False +prop_checkUnnecessaryArithmeticExpansionIndex1 = verify checkUnnecessaryArithmeticExpansionIndex "a[$((1+1))]=n" +prop_checkUnnecessaryArithmeticExpansionIndex2 = verifyNot checkUnnecessaryArithmeticExpansionIndex "a[1+1]=n" +prop_checkUnnecessaryArithmeticExpansionIndex3 = verifyNot checkUnnecessaryArithmeticExpansionIndex "a[$(echo $((1+1)))]=n" +checkUnnecessaryArithmeticExpansionIndex params t = + case t of + T_Assignment _ mode var [TA_Sequence _ [ TA_Expansion _ [expansion@(T_DollarArithmetic id _)]]] val -> + styleWithFix id 2321 "Array indices are already arithmetic contexts. Prefer removing the $(( and ))." $ fix id + _ -> return () + + where + fix id = + fixWith [ + replaceStart id params 3 "", -- Remove "$((" + replaceEnd id params 2 "" -- Remove "))" + ] + + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) From 52dac51cd4d919667e3a2d3d3feec1d516464392 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 24 Jul 2022 14:06:01 -0700 Subject: [PATCH 558/763] SC2323: Warn about redundant parens in a[(x+1)] and $(( ((x)) )) (ref: #1666) --- CHANGELOG.md | 2 ++ src/ShellCheck/AST.hs | 4 +++- src/ShellCheck/Analytics.hs | 33 +++++++++++++++++++++++++++++++++ src/ShellCheck/CFG.hs | 1 + src/ShellCheck/Parser.hs | 4 +++- 5 files changed, 42 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 669579f..cef16f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ - SC2318: Warn about backreferences in 'declare x=1 y=$x' - SC2319/SC2320: Warn when $? refers to echo/printf/[ ]/[[ ]]/test - SC2321: Suggest removing $((..)) in array[$((idx))]=val +- SC2322: Suggest collapsing double parentheses in arithmetic contexts +- SC2323: Suggest removing wrapping parentheses in a[(x+1)]=val ### Fixed - SC2086: Now uses DFA to make more accurate predictions about values diff --git a/src/ShellCheck/AST.hs b/src/ShellCheck/AST.hs index 2cd2f6f..ca5007a 100644 --- a/src/ShellCheck/AST.hs +++ b/src/ShellCheck/AST.hs @@ -45,6 +45,7 @@ data InnerToken t = | Inner_TA_Variable String [t] | Inner_TA_Expansion [t] | Inner_TA_Sequence [t] + | Inner_TA_Parenthesis t | Inner_TA_Trinary t t t | Inner_TA_Unary String t | Inner_TC_And ConditionType String t t @@ -204,6 +205,7 @@ pattern T_Annotation id anns t = OuterToken id (Inner_T_Annotation anns t) pattern T_Arithmetic id c = OuterToken id (Inner_T_Arithmetic c) pattern T_Array id t = OuterToken id (Inner_T_Array t) pattern TA_Sequence id l = OuterToken id (Inner_TA_Sequence l) +pattern TA_Parentesis id t = OuterToken id (Inner_TA_Parenthesis t) pattern T_Assignment id mode var indices value = OuterToken id (Inner_T_Assignment mode var indices value) pattern TA_Trinary id t1 t2 t3 = OuterToken id (Inner_TA_Trinary t1 t2 t3) pattern TA_Unary id op t1 = OuterToken id (Inner_TA_Unary op t1) @@ -256,7 +258,7 @@ pattern T_Subshell id l = OuterToken id (Inner_T_Subshell l) pattern T_UntilExpression id c l = OuterToken id (Inner_T_UntilExpression c l) pattern T_WhileExpression id c l = OuterToken id (Inner_T_WhileExpression c l) -{-# COMPLETE T_AND_IF, T_Bang, T_Case, TC_Empty, T_CLOBBER, T_DGREAT, T_DLESS, T_DLESSDASH, T_Do, T_DollarSingleQuoted, T_Done, T_DSEMI, T_Elif, T_Else, T_EOF, T_Esac, T_Fi, T_For, T_Glob, T_GREATAND, T_Greater, T_If, T_In, T_Lbrace, T_Less, T_LESSAND, T_LESSGREAT, T_Literal, T_Lparen, T_NEWLINE, T_OR_IF, T_ParamSubSpecialChar, T_Pipe, T_Rbrace, T_Rparen, T_Select, T_Semi, T_SingleQuoted, T_Then, T_UnparsedIndex, T_Until, T_While, TA_Assignment, TA_Binary, TA_Expansion, T_AndIf, T_Annotation, T_Arithmetic, T_Array, TA_Sequence, T_Assignment, TA_Trinary, TA_Unary, TA_Variable, T_Backgrounded, T_Backticked, T_Banged, T_BatsTest, T_BraceExpansion, T_BraceGroup, TC_And, T_CaseExpression, TC_Binary, TC_Group, TC_Nullary, T_Condition, T_CoProcBody, T_CoProc, TC_Or, TC_Unary, T_DollarArithmetic, T_DollarBraceCommandExpansion, T_DollarBraced, T_DollarBracket, T_DollarDoubleQuoted, T_DollarExpansion, T_DoubleQuoted, T_Extglob, T_FdRedirect, T_ForArithmetic, T_ForIn, T_Function, T_HereDoc, T_HereString, T_IfExpression, T_Include, T_IndexedElement, T_IoDuplicate, T_IoFile, T_NormalWord, T_OrIf, T_Pipeline, T_ProcSub, T_Redirecting, T_Script, T_SelectIn, T_SimpleCommand, T_SourceCommand, T_Subshell, T_UntilExpression, T_WhileExpression #-} +{-# COMPLETE T_AND_IF, T_Bang, T_Case, TC_Empty, T_CLOBBER, T_DGREAT, T_DLESS, T_DLESSDASH, T_Do, T_DollarSingleQuoted, T_Done, T_DSEMI, T_Elif, T_Else, T_EOF, T_Esac, T_Fi, T_For, T_Glob, T_GREATAND, T_Greater, T_If, T_In, T_Lbrace, T_Less, T_LESSAND, T_LESSGREAT, T_Literal, T_Lparen, T_NEWLINE, T_OR_IF, T_ParamSubSpecialChar, T_Pipe, T_Rbrace, T_Rparen, T_Select, T_Semi, T_SingleQuoted, T_Then, T_UnparsedIndex, T_Until, T_While, TA_Assignment, TA_Binary, TA_Expansion, T_AndIf, T_Annotation, T_Arithmetic, T_Array, TA_Sequence, TA_Parentesis, T_Assignment, TA_Trinary, TA_Unary, TA_Variable, T_Backgrounded, T_Backticked, T_Banged, T_BatsTest, T_BraceExpansion, T_BraceGroup, TC_And, T_CaseExpression, TC_Binary, TC_Group, TC_Nullary, T_Condition, T_CoProcBody, T_CoProc, TC_Or, TC_Unary, T_DollarArithmetic, T_DollarBraceCommandExpansion, T_DollarBraced, T_DollarBracket, T_DollarDoubleQuoted, T_DollarExpansion, T_DoubleQuoted, T_Extglob, T_FdRedirect, T_ForArithmetic, T_ForIn, T_Function, T_HereDoc, T_HereString, T_IfExpression, T_Include, T_IndexedElement, T_IoDuplicate, T_IoFile, T_NormalWord, T_OrIf, T_Pipeline, T_ProcSub, T_Redirecting, T_Script, T_SelectIn, T_SimpleCommand, T_SourceCommand, T_Subshell, T_UntilExpression, T_WhileExpression #-} instance Eq Token where OuterToken _ a == OuterToken _ b = a == b diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index e92f3ff..eed2d25 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -207,6 +207,7 @@ nodeChecks = [ ,checkSpacefulnessCfg ,checkOverwrittenExitCode ,checkUnnecessaryArithmeticExpansionIndex + ,checkUnnecessaryParens ] optionalChecks = map fst optionalTreeChecks @@ -3280,6 +3281,7 @@ checkReturnAgainstZero params token = _:next@(TA_Unary _ "!" _):_ -> isOnlyTestInCommand next _:next@(TC_Group {}):_ -> isOnlyTestInCommand next _:next@(TA_Sequence _ [_]):_ -> isOnlyTestInCommand next + _:next@(TA_Parentesis _ _):_ -> isOnlyTestInCommand next _ -> False -- TODO: Do better $? tracking and filter on whether @@ -4931,5 +4933,36 @@ checkUnnecessaryArithmeticExpansionIndex params t = ] +prop_checkUnnecessaryParens1 = verify checkUnnecessaryParens "echo $(( ((1+1)) ))" +prop_checkUnnecessaryParens2 = verify checkUnnecessaryParens "x[((1+1))+1]=1" +prop_checkUnnecessaryParens3 = verify checkUnnecessaryParens "x[(1+1)]=1" +prop_checkUnnecessaryParens4 = verify checkUnnecessaryParens "$(( (x) ))" +prop_checkUnnecessaryParens5 = verify checkUnnecessaryParens "(( (x) ))" +prop_checkUnnecessaryParens6 = verifyNot checkUnnecessaryParens "x[(1+1)+1]=1" +prop_checkUnnecessaryParens7 = verifyNot checkUnnecessaryParens "(( (1*1)+1 ))" +prop_checkUnnecessaryParens8 = verifyNot checkUnnecessaryParens "(( (1)+1 ))" +checkUnnecessaryParens params t = + case t of + T_DollarArithmetic _ t -> checkLeading "$(( (x) )) is the same as $(( x ))" t + T_ForArithmetic _ x y z _ -> mapM_ (checkLeading "for (((x); (y); (z))) is the same as for ((x; y; z))") [x,y,z] + T_Assignment _ _ _ [t] _ -> checkLeading "a[(x)] is the same as a[x]" t + T_Arithmetic _ t -> checkLeading "(( (x) )) is the same as (( x ))" t + TA_Parentesis _ (TA_Sequence _ [ TA_Parentesis id _ ]) -> + styleWithFix id 2322 "In arithmetic contexts, ((x)) is the same as (x). Prefer only one layer of parentheses." $ fix id + _ -> return () + where + + checkLeading str t = + case t of + TA_Sequence _ [TA_Parentesis id _ ] -> styleWithFix id 2323 (str ++ ". Prefer not wrapping in additional parentheses.") $ fix id + _ -> return () + + fix id = + fixWith [ + replaceStart id params 1 "", -- Remove "(" + replaceEnd id params 1 "" -- Remove ")" + ] + + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) diff --git a/src/ShellCheck/CFG.hs b/src/ShellCheck/CFG.hs index 1085d8f..6f6d4f1 100644 --- a/src/ShellCheck/CFG.hs +++ b/src/ShellCheck/CFG.hs @@ -479,6 +479,7 @@ build t = do TA_Binary _ _ a b -> sequentially [a,b] TA_Expansion _ list -> sequentially list TA_Sequence _ list -> sequentially list + TA_Parentesis _ t -> build t TA_Trinary _ cond a b -> do condition <- build cond diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index e6a2999..0dd6621 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -821,11 +821,13 @@ readArithmeticContents = return $ TA_Expansion id pieces readGroup = do + start <- startSpan char '(' s <- readSequence char ')' + id <- endSpan start spacing - return s + return $ TA_Parentesis id s readArithTerm = readGroup <|> readVariable <|> readExpansion From 982681fc05ca8db887431691d1cc40633e20d828 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 24 Jul 2022 14:30:31 -0700 Subject: [PATCH 559/763] Add unit test to ensure SC2321 does not trigger on associative arrays --- src/ShellCheck/Analytics.hs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index eed2d25..8499d8d 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -4919,6 +4919,7 @@ checkOverwrittenExitCode params t = prop_checkUnnecessaryArithmeticExpansionIndex1 = verify checkUnnecessaryArithmeticExpansionIndex "a[$((1+1))]=n" prop_checkUnnecessaryArithmeticExpansionIndex2 = verifyNot checkUnnecessaryArithmeticExpansionIndex "a[1+1]=n" prop_checkUnnecessaryArithmeticExpansionIndex3 = verifyNot checkUnnecessaryArithmeticExpansionIndex "a[$(echo $((1+1)))]=n" +prop_checkUnnecessaryArithmeticExpansionIndex4 = verifyNot checkUnnecessaryArithmeticExpansionIndex "declare -A a; a[$((1+1))]=val" checkUnnecessaryArithmeticExpansionIndex params t = case t of T_Assignment _ mode var [TA_Sequence _ [ TA_Expansion _ [expansion@(T_DollarArithmetic id _)]]] val -> From f1148b8b41087dba8441a2c79980441522e3b23b Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 25 Jul 2022 10:00:50 -0700 Subject: [PATCH 560/763] Include postdominators in CFGResult --- src/ShellCheck/CFG.hs | 86 ++++++++++++++++++++++++++++++++--- src/ShellCheck/CFGAnalysis.hs | 5 +- 2 files changed, 84 insertions(+), 7 deletions(-) diff --git a/src/ShellCheck/CFG.hs b/src/ShellCheck/CFG.hs index 6f6d4f1..ad05e93 100644 --- a/src/ShellCheck/CFG.hs +++ b/src/ShellCheck/CFG.hs @@ -54,6 +54,8 @@ import qualified Data.Set as S import Control.Monad.RWS.Lazy import Data.Graph.Inductive.Graph import Data.Graph.Inductive.Query.DFS +import Data.Graph.Inductive.Basic +import Data.Graph.Inductive.Query.Dominators import Data.Graph.Inductive.PatriciaTree as G import Debug.Trace -- STRIP @@ -171,9 +173,11 @@ data CFGResult = CFGResult { -- Map from Id to nominal start&end node (i.e. assuming normal execution without exits) cfIdToRange :: M.Map Id (Node, Node), -- A set of all nodes belonging to an Id, recursively - cfIdToNodes :: M.Map Id (S.Set Node) + cfIdToNodes :: M.Map Id (S.Set Node), + -- A map to nodes that the given node postdominates + cfPostDominators :: M.Map Node (S.Set Node) } - deriving (Show) + deriving (Show, Generic, NFData) buildGraph :: CFGParameters -> Token -> CFGResult buildGraph params root = @@ -183,12 +187,20 @@ buildGraph params root = -- renumberTopologically $ removeUnnecessaryStructuralNodes base - in - CFGResult { + + idToRange = M.fromList mapping + isRealEdge (from, to, edge) = case edge of CFEFlow -> True; _ -> False + onlyRealEdges = filter isRealEdge edges + (_, mainExit) = fromJust $ M.lookup (getId root) idToRange + + result = CFGResult { cfGraph = mkGraph nodes edges, - cfIdToRange = M.fromList mapping, - cfIdToNodes = M.fromListWith S.union $ map (\(id, n) -> (id, S.singleton n)) association + cfIdToRange = idToRange, + cfIdToNodes = M.fromListWith S.union $ map (\(id, n) -> (id, S.singleton n)) association, + cfPostDominators = findPostDominators mainExit $ mkGraph nodes onlyRealEdges } + in + deepseq result result remapGraph :: M.Map Node Node -> CFW -> CFW remapGraph remap (nodes, edges, mapping, assoc) = @@ -1190,5 +1202,67 @@ tokenToParts t = -- Check if getLiteralString can handle it, if not it's unknown _ -> [maybe CFStringUnknown CFStringLiteral $ getLiteralString t] + +-- Change all subshell invocations to instead link directly to their contents. +-- This is used for producing dominator trees. +inlineSubshells :: CFGraph -> CFGraph +inlineSubshells graph = relinkedGraph + where + subshells = ufold find [] graph + find (incoming, node, label, outgoing) acc = + case label of + CFExecuteSubshell _ start end -> (node, label, start, end, incoming, outgoing):acc + _ -> acc + + relinkedGraph = foldl' relink graph subshells + relink graph (node, label, start, end, incoming, outgoing) = + let + -- Link CFExecuteSubshell to the CFEntryPoint + subshellToStart = (incoming, node, label, [(CFEFlow, start)]) + -- Link the subshell exit to the + endToNexts = (endIncoming, endNode, endLabel, outgoing) + (endIncoming, endNode, endLabel, _) = context graph end + in + subshellToStart & (endToNexts & graph) + +findEntryNodes :: CFGraph -> [Node] +findEntryNodes graph = ufold find [] graph + where + find (incoming, node, label, _) list = + case label of + CFEntryPoint {} | null incoming -> node:list + _ -> list + +findDominators main graph = asSetMap + where + inlined = inlineSubshells graph + entryNodes = main : findEntryNodes graph + asLists = concatMap (dom inlined) entryNodes + asSetMap = M.fromList $ map (\(node, list) -> (node, S.fromList list)) asLists + +findTerminalNodes :: CFGraph -> [Node] +findTerminalNodes graph = ufold find [] graph + where + find (_, node, label, _) list = + case label of + CFUnresolvedExit -> node:list + CFApplyEffects effects -> f effects list + _ -> list + + f [] list = list + f (IdTagged _ (CFDefineFunction _ id start end):rest) list = f rest (end:list) + f (_:rest) list = f rest list + +findPostDominators :: Node -> CFGraph -> M.Map Node (S.Set Node) +findPostDominators mainexit graph = asSetMap + where + inlined = inlineSubshells graph + terminals = findTerminalNodes inlined + (incoming, _, label, outgoing) = context graph mainexit + withExitEdges = (incoming ++ map (\c -> (CFEFlow, c)) terminals, mainexit, label, outgoing) & inlined + reversed = grev withExitEdges + postDoms = dom reversed mainexit + asSetMap = M.fromList $ map (\(node, list) -> (node, S.fromList list)) postDoms + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) diff --git a/src/ShellCheck/CFGAnalysis.hs b/src/ShellCheck/CFGAnalysis.hs index bb90860..dc0a4b1 100644 --- a/src/ShellCheck/CFGAnalysis.hs +++ b/src/ShellCheck/CFGAnalysis.hs @@ -99,6 +99,7 @@ data CFGAnalysis = CFGAnalysis { graph :: CFGraph, tokenToRange :: M.Map Id (Node, Node), tokenToNodes :: M.Map Id (S.Set Node), + postDominators :: M.Map Node (S.Set Node), nodeToData :: M.Map Node (ProgramState, ProgramState) } deriving (Show, Generic, NFData) @@ -1304,7 +1305,8 @@ analyzeControlFlow params t = graph = cfGraph cfg, tokenToRange = cfIdToRange cfg, tokenToNodes = cfIdToNodes cfg, - nodeToData = nodeToData + nodeToData = nodeToData, + postDominators = cfPostDominators cfg } @@ -1355,5 +1357,6 @@ analyzeStragglers ctx state stragglers = do transferFunctionValue ctx def + return [] runTests = $quickCheckAll From e9784fa9a77f6f018997f5d164148422f7da0b33 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 25 Jul 2022 11:57:04 -0700 Subject: [PATCH 561/763] Refine #2544 to not warn when $? postdominates [ ] (fixes #2544) --- src/ShellCheck/Analytics.hs | 13 +++++++++++-- src/ShellCheck/CFGAnalysis.hs | 10 ++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 8499d8d..42d5a9e 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -4884,7 +4884,11 @@ checkCommandIsUnreachable params t = prop_checkOverwrittenExitCode1 = verify checkOverwrittenExitCode "x; [ $? -eq 1 ] || [ $? -eq 2 ]" prop_checkOverwrittenExitCode2 = verifyNot checkOverwrittenExitCode "x; [ $? -eq 1 ]" prop_checkOverwrittenExitCode3 = verify checkOverwrittenExitCode "x; echo \"Exit is $?\"; [ $? -eq 0 ]" -prop_checkOverwrittenExitCode4 = verifyNot checkOverwrittenExitCode "x; [ $? -eq 0 ]" +prop_checkOverwrittenExitCode4 = verifyNot checkOverwrittenExitCode "x; [ $? -eq 0 ] && echo Success" +prop_checkOverwrittenExitCode5 = verify checkOverwrittenExitCode "x; if [ $? -eq 0 ]; then var=$?; fi" +prop_checkOverwrittenExitCode6 = verify checkOverwrittenExitCode "x; [ $? -gt 0 ] && fail=$?" +prop_checkOverwrittenExitCode7 = verifyNot checkOverwrittenExitCode "[ 1 -eq 2 ]; status=$?" +prop_checkOverwrittenExitCode8 = verifyNot checkOverwrittenExitCode "[ 1 -eq 2 ]; exit $?" checkOverwrittenExitCode params t = case t of T_DollarBraced id _ val | getLiteralString val == Just "?" -> check id @@ -4898,7 +4902,7 @@ checkOverwrittenExitCode params t = let idToToken = idMap params exitCodeTokens <- sequence $ map (\k -> Map.lookup k idToToken) $ S.toList exitCodeIds return $ do - when (all isCondition exitCodeTokens) $ + when (all isCondition exitCodeTokens && not (usedUnconditionally t exitCodeIds)) $ warn id 2319 "This $? refers to a condition, not a command. Assign to a variable to avoid it being overwritten." when (all isPrinting exitCodeTokens) $ warn id 2320 "This $? refers to echo/printf, not a previous command. Assign to variable to avoid it being overwritten." @@ -4909,6 +4913,11 @@ checkOverwrittenExitCode params t = T_SimpleCommand {} -> getCommandName t == Just "test" _ -> False + -- If we don't do anything based on the condition, assume we wanted the condition itself + -- This helps differentiate `x; [ $? -gt 0 ] && exit $?` vs `[ cond ]; exit $?` + usedUnconditionally t testIds = + all (\c -> CF.doesPostDominate (cfgAnalysis params) (getId t) c) testIds + isPrinting t = case getCommandBasename t of Just "echo" -> True diff --git a/src/ShellCheck/CFGAnalysis.hs b/src/ShellCheck/CFGAnalysis.hs index dc0a4b1..ff88810 100644 --- a/src/ShellCheck/CFGAnalysis.hs +++ b/src/ShellCheck/CFGAnalysis.hs @@ -56,6 +56,7 @@ module ShellCheck.CFGAnalysis ( ,SpaceStatus (..) ,getIncomingState ,getOutgoingState + ,doesPostDominate ,ShellCheck.CFGAnalysis.runTests -- STRIP ) where @@ -140,6 +141,15 @@ getOutgoingState analysis id = do (start,end) <- M.lookup id $ tokenToRange analysis snd <$> M.lookup end (nodeToData analysis) +-- Conveniently determine whether one node postdominates another, +-- i.e. whether 'target' always unconditionally runs after 'base'. +doesPostDominate :: CFGAnalysis -> Id -> Id -> Bool +doesPostDominate analysis target base = fromMaybe False $ do + (_, baseEnd) <- M.lookup base $ tokenToRange analysis + (targetStart, _) <- M.lookup target $ tokenToRange analysis + postDoms <- M.lookup baseEnd $ postDominators analysis + return $ S.member targetStart postDoms + getDataForNode analysis node = M.lookup node $ nodeToData analysis -- The current state of data flow at a point in the program, potentially as a diff From c57e447c89a9ba64bd717560edbeb2192bb7b92a Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 26 Jul 2022 09:46:07 -0700 Subject: [PATCH 562/763] Correctly discard overlapping fixes in diff output (fixes #2370) --- src/ShellCheck/Formatter/Diff.hs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/ShellCheck/Formatter/Diff.hs b/src/ShellCheck/Formatter/Diff.hs index 197b3af..15d00d7 100644 --- a/src/ShellCheck/Formatter/Diff.hs +++ b/src/ShellCheck/Formatter/Diff.hs @@ -203,10 +203,9 @@ formatDoc color (DiffDoc name lf regions) = buildFixMap :: [Fix] -> M.Map String Fix buildFixMap fixes = perFile where - splitFixes = concatMap splitFixByFile fixes + splitFixes = splitFixByFile $ mconcat fixes perFile = groupByMap (posFile . repStartPos . head . fixReplacements) splitFixes --- There are currently no multi-file fixes, but let's handle it anyways splitFixByFile :: Fix -> [Fix] splitFixByFile fix = map makeFix $ groupBy sameFile (fixReplacements fix) where From b5f5e6347d59be6b26ce8761646fe95ac7b2f3c7 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 26 Jul 2022 10:42:01 -0700 Subject: [PATCH 563/763] Discard next rather than existing fixes when they overlap --- src/ShellCheck/Fixer.hs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ShellCheck/Fixer.hs b/src/ShellCheck/Fixer.hs index 2376842..43a97ab 100644 --- a/src/ShellCheck/Fixer.hs +++ b/src/ShellCheck/Fixer.hs @@ -87,6 +87,7 @@ instance Ranged Replacement where instance Monoid Fix where mempty = newFix mappend = (<>) + mconcat = foldl mappend mempty -- fold left to right since <> discards right on overlap instance Semigroup Fix where f1 <> f2 = From 4a27c9a8d50996438ff4853206be674e28069386 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 26 Jul 2022 15:33:25 -0700 Subject: [PATCH 564/763] Fix overlap check --- src/ShellCheck/Fixer.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Fixer.hs b/src/ShellCheck/Fixer.hs index 43a97ab..358dec9 100644 --- a/src/ShellCheck/Fixer.hs +++ b/src/ShellCheck/Fixer.hs @@ -36,7 +36,7 @@ class Ranged a where end :: a -> Position overlap :: a -> a -> Bool overlap x y = - (yStart >= xStart && yStart < xEnd) || (yStart < xStart && yEnd > xStart) + xEnd > yStart && yEnd > xStart where yStart = start y yEnd = end y From a30ac402eb39d3e60cd3a64abd48d00c85d49bab Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 27 Jul 2022 11:29:55 -0700 Subject: [PATCH 565/763] Don't use & for updates as result is unspecified This fixes `Prelude.foldl1: empty list []` when script has `( exit )` --- src/ShellCheck/CFG.hs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/CFG.hs b/src/ShellCheck/CFG.hs index ad05e93..39747cf 100644 --- a/src/ShellCheck/CFG.hs +++ b/src/ShellCheck/CFG.hs @@ -1203,6 +1203,9 @@ tokenToParts t = _ -> [maybe CFStringUnknown CFStringLiteral $ getLiteralString t] +-- Like & but well defined when the node already exists +safeUpdate ctx@(_,node,_,_) graph = ctx & (delNode node graph) + -- Change all subshell invocations to instead link directly to their contents. -- This is used for producing dominator trees. inlineSubshells :: CFGraph -> CFGraph @@ -1223,7 +1226,7 @@ inlineSubshells graph = relinkedGraph endToNexts = (endIncoming, endNode, endLabel, outgoing) (endIncoming, endNode, endLabel, _) = context graph end in - subshellToStart & (endToNexts & graph) + subshellToStart `safeUpdate` (endToNexts `safeUpdate` graph) findEntryNodes :: CFGraph -> [Node] findEntryNodes graph = ufold find [] graph @@ -1259,7 +1262,7 @@ findPostDominators mainexit graph = asSetMap inlined = inlineSubshells graph terminals = findTerminalNodes inlined (incoming, _, label, outgoing) = context graph mainexit - withExitEdges = (incoming ++ map (\c -> (CFEFlow, c)) terminals, mainexit, label, outgoing) & inlined + withExitEdges = (incoming ++ map (\c -> (CFEFlow, c)) terminals, mainexit, label, outgoing) `safeUpdate` inlined reversed = grev withExitEdges postDoms = dom reversed mainexit asSetMap = M.fromList $ map (\(node, list) -> (node, S.fromList list)) postDoms From 3ce310e939427216e461681e0a71c4883ff5bf03 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 27 Jul 2022 14:25:19 -0700 Subject: [PATCH 566/763] Plug space leaks when processing multiple files --- shellcheck.hs | 2 +- src/ShellCheck/Formatter/JSON.hs | 3 ++- src/ShellCheck/Formatter/JSON1.hs | 3 ++- src/ShellCheck/Formatter/TTY.hs | 3 ++- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/shellcheck.hs b/shellcheck.hs index bf70445..a525251 100644 --- a/shellcheck.hs +++ b/shellcheck.hs @@ -225,7 +225,7 @@ runFormatter sys format options files = do f :: Status -> FilePath -> IO Status f status file = do newStatus <- process file `catch` handler file - return $ status `mappend` newStatus + return $! status `mappend` newStatus handler :: FilePath -> IOException -> IO Status handler file e = reportFailure file (show e) reportFailure file str = do diff --git a/src/ShellCheck/Formatter/JSON.hs b/src/ShellCheck/Formatter/JSON.hs index 7c26421..6b38532 100644 --- a/src/ShellCheck/Formatter/JSON.hs +++ b/src/ShellCheck/Formatter/JSON.hs @@ -23,6 +23,7 @@ module ShellCheck.Formatter.JSON (format) where import ShellCheck.Interface import ShellCheck.Formatter.Format +import Control.DeepSeq import Data.Aeson import Data.IORef import Data.Monoid @@ -103,7 +104,7 @@ collectResult ref cr sys = mapM_ f groups comments = crComments cr groups = groupWith sourceFile comments f :: [PositionedComment] -> IO () - f group = modifyIORef ref (\x -> comments ++ x) + f group = deepseq comments $ modifyIORef ref (\x -> comments ++ x) finish ref = do list <- readIORef ref diff --git a/src/ShellCheck/Formatter/JSON1.hs b/src/ShellCheck/Formatter/JSON1.hs index 54aad34..2169bf6 100644 --- a/src/ShellCheck/Formatter/JSON1.hs +++ b/src/ShellCheck/Formatter/JSON1.hs @@ -23,6 +23,7 @@ module ShellCheck.Formatter.JSON1 (format) where import ShellCheck.Interface import ShellCheck.Formatter.Format +import Control.DeepSeq import Data.Aeson import Data.IORef import Data.Monoid @@ -120,7 +121,7 @@ collectResult ref cr sys = mapM_ f groups result <- siReadFile sys (Just True) filename let contents = either (const "") id result let comments' = makeNonVirtual comments contents - modifyIORef ref (\x -> comments' ++ x) + deepseq comments' $ modifyIORef ref (\x -> comments' ++ x) finish ref = do list <- readIORef ref diff --git a/src/ShellCheck/Formatter/TTY.hs b/src/ShellCheck/Formatter/TTY.hs index 8dd90d4..e28696c 100644 --- a/src/ShellCheck/Formatter/TTY.hs +++ b/src/ShellCheck/Formatter/TTY.hs @@ -23,6 +23,7 @@ import ShellCheck.Fixer import ShellCheck.Interface import ShellCheck.Formatter.Format +import Control.DeepSeq import Control.Monad import Data.Array import Data.Foldable @@ -88,7 +89,7 @@ rankError err = (ranking, cSeverity $ pcComment err, cCode $ pcComment err) appendComments errRef comments max = do previous <- readIORef errRef let current = map (\x -> (rankError x, cCode $ pcComment x, cMessage $ pcComment x)) comments - writeIORef errRef . take max . nubBy equal . sort $ previous ++ current + writeIORef errRef $! force . take max . nubBy equal . sort $ previous ++ current where fst3 (x,_,_) = x equal x y = fst3 x == fst3 y From f4409122799d7967ff843259bc3a51e2fcad2cef Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 27 Jul 2022 19:47:37 -0700 Subject: [PATCH 567/763] Refactor to not generate Parameters twice --- src/ShellCheck/ASTLib.hs | 4 ++++ src/ShellCheck/Analytics.hs | 39 +++++++++++++++---------------------- src/ShellCheck/Analyzer.hs | 4 ++-- src/ShellCheck/Checker.hs | 5 +++-- 4 files changed, 25 insertions(+), 27 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 7b4f9e5..56903ee 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -898,6 +898,10 @@ isClosingFileOp op = T_IoDuplicate _ (T_LESSAND _) "-" -> True _ -> False +getEnableDirectives root = + case root of + T_Annotation _ list _ -> [s | EnableComment s <- list] + _ -> [] return [] runTests = $quickCheckAll diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 42d5a9e..e878dc4 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -19,7 +19,7 @@ -} {-# LANGUAGE TemplateHaskell #-} {-# LANGUAGE FlexibleContexts #-} -module ShellCheck.Analytics (runAnalytics, optionalChecks, ShellCheck.Analytics.runTests) where +module ShellCheck.Analytics (checker, optionalChecks, ShellCheck.Analytics.runTests) where import ShellCheck.AST import ShellCheck.ASTLib @@ -71,29 +71,22 @@ treeChecks = [ ,checkArrayValueUsedAsIndex ] -runAnalytics :: AnalysisSpec -> [TokenComment] -runAnalytics options = - runList options treeChecks ++ runList options optionalChecks +checker spec params = mkChecker spec params treeChecks + +mkChecker spec params checks = + Checker { + perScript = \(Root root) -> do + tell $ concatMap (\f -> f params root) all, + perToken = const $ return () + } where - root = asScript options - optionals = getEnableDirectives root ++ asOptionalChecks options - optionalChecks = - if "all" `elem` optionals + all = checks ++ optionals + optionalKeys = asOptionalChecks spec + optionals = + if "all" `elem` optionalKeys then map snd optionalTreeChecks - else mapMaybe (\c -> Map.lookup c optionalCheckMap) optionals + else mapMaybe (\c -> Map.lookup c optionalCheckMap) optionalKeys -runList :: AnalysisSpec -> [Parameters -> Token -> [TokenComment]] - -> [TokenComment] -runList spec list = notes - where - root = asScript spec - params = makeParameters spec - notes = concatMap (\f -> f params root) list - -getEnableDirectives root = - case root of - T_Annotation _ list _ -> [s | EnableComment s <- list] - _ -> [] checkList l t = concatMap (\f -> f t) l @@ -318,12 +311,12 @@ producesComments f s = not . null <$> runAndGetComments f s runAndGetComments f s = do let pr = pScript s - prRoot pr + root <- prRoot pr let spec = defaultSpec pr let params = makeParameters spec return $ filterByAnnotation spec params $ - runList spec [f] + f params root -- Copied from https://wiki.haskell.org/Edit_distance dist :: Eq a => [a] -> [a] -> Int diff --git a/src/ShellCheck/Analyzer.hs b/src/ShellCheck/Analyzer.hs index ff2e457..06b6e53 100644 --- a/src/ShellCheck/Analyzer.hs +++ b/src/ShellCheck/Analyzer.hs @@ -35,13 +35,13 @@ analyzeScript :: AnalysisSpec -> AnalysisResult analyzeScript spec = newAnalysisResult { arComments = filterByAnnotation spec params . nub $ - runAnalytics spec - ++ runChecker params (checkers spec params) + runChecker params (checkers spec params) } where params = makeParameters spec checkers spec params = mconcat $ map ($ params) [ + ShellCheck.Analytics.checker spec, ShellCheck.Checks.Commands.checker spec, ShellCheck.Checks.ControlFlow.checker spec, ShellCheck.Checks.Custom.checker, diff --git a/src/ShellCheck/Checker.hs b/src/ShellCheck/Checker.hs index ef8182f..db793f1 100644 --- a/src/ShellCheck/Checker.hs +++ b/src/ShellCheck/Checker.hs @@ -20,9 +20,10 @@ {-# LANGUAGE TemplateHaskell #-} module ShellCheck.Checker (checkScript, ShellCheck.Checker.runTests) where +import ShellCheck.Analyzer +import ShellCheck.ASTLib import ShellCheck.Interface import ShellCheck.Parser -import ShellCheck.Analyzer import Data.Either import Data.Functor @@ -85,7 +86,7 @@ checkScript sys spec = do asCheckSourced = csCheckSourced spec, asExecutionMode = Executed, asTokenPositions = tokenPositions, - asOptionalChecks = csOptionalChecks spec + asOptionalChecks = getEnableDirectives root ++ csOptionalChecks spec } where as = newAnalysisSpec root let analysisMessages = maybe [] From d0dd81e1faa506232193ad91030dd2cb9f2d4a66 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 28 Jul 2022 08:56:44 -0700 Subject: [PATCH 568/763] Allow quoting values in directives (fixes #2517) --- CHANGELOG.md | 1 + shellcheck.1.md | 3 +++ src/ShellCheck/Checker.hs | 12 ++++++++++++ src/ShellCheck/Parser.hs | 23 +++++++++++++++++------ 4 files changed, 33 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cef16f4..c363eb5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ based on control flow rather than just syntax. Existing checks will gradually start using it, which may cause them to trigger differently (but more accurately). +- Values in directives/shellcheckrc can now be quoted with '' or "" ## v0.8.0 - 2021-11-06 diff --git a/shellcheck.1.md b/shellcheck.1.md index 146d791..c345a2b 100644 --- a/shellcheck.1.md +++ b/shellcheck.1.md @@ -282,6 +282,9 @@ Here is an example `.shellcheckrc`: source-path=SCRIPTDIR source-path=/mnt/chroot + # Since 0.9.0, values can be quoted with '' or "" to allow spaces + source-path="My Documents/scripts" + # Allow opening any 'source'd file, even if not specified as input external-sources=true diff --git a/src/ShellCheck/Checker.hs b/src/ShellCheck/Checker.hs index db793f1..6518e0d 100644 --- a/src/ShellCheck/Checker.hs +++ b/src/ShellCheck/Checker.hs @@ -244,6 +244,9 @@ prop_canStripPrefixAndSource2 = prop_canSourceDynamicWhenRedirected = null $ checkWithIncludes [("lib", "")] "#shellcheck source=lib\n. \"$1\"" +prop_canRedirectWithSpaces = + null $ checkWithIncludes [("my file", "")] "#shellcheck source=\"my file\"\n. \"$1\"" + prop_recursiveAnalysis = [2086] == checkRecursive [("lib", "echo $1")] "source lib" @@ -413,6 +416,15 @@ prop_sourcePathAddsAnnotation = result == [2086] csCheckSourced = True } +prop_sourcePathWorksWithSpaces = result == [2086] + where + f "dir/myscript" _ ["my path"] "lib" = return "foo/lib" + result = checkWithIncludesAndSourcePath [("foo/lib", "echo $1")] f emptyCheckSpec { + csScript = "#!/bin/bash\n# shellcheck source-path='my path'\nsource lib", + csFilename = "dir/myscript", + csCheckSourced = True + } + prop_sourcePathRedirectsDirective = result == [2086] where f "dir/myscript" _ _ "lib" = return "foo/lib" diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 0dd6621..4ff45ed 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -992,6 +992,10 @@ prop_readAnnotation5 = isOk readAnnotation "# shellcheck disable=SC2002 # All ca prop_readAnnotation6 = isOk readAnnotation "# shellcheck disable=SC1234 # shellcheck foo=bar\n" prop_readAnnotation7 = isOk readAnnotation "# shellcheck disable=SC1000,SC2000-SC3000,SC1001\n" prop_readAnnotation8 = isOk readAnnotation "# shellcheck disable=all\n" +prop_readAnnotation9 = isOk readAnnotation "# shellcheck source='foo bar' source-path=\"baz etc\"\n" +prop_readAnnotation10 = isOk readAnnotation "# shellcheck disable='SC1234,SC2345' enable=\"foo\" shell='bash'\n" +prop_readAnnotation11 = isOk (readAnnotationWithoutPrefix False) "external-sources='true'" + readAnnotation = called "shellcheck directive" $ do try readAnnotationPrefix many1 linewhitespace @@ -1007,12 +1011,19 @@ readAnnotationWithoutPrefix sandboxed = do many linewhitespace return $ concat values where + plainOrQuoted p = quoted p <|> p + quoted p = do + c <- oneOf "'\"" + start <- getPosition + str <- many1 $ noneOf (c:"\n") + char c <|> fail "Missing terminating quote for directive." + subParse start p str readKey = do keyPos <- getPosition key <- many1 (letter <|> char '-') char '=' <|> fail "Expected '=' after directive key" annotations <- case key of - "disable" -> readElement `sepBy` char ',' + "disable" -> plainOrQuoted $ readElement `sepBy` char ',' where readElement = readRange <|> readAll readAll = do @@ -1027,21 +1038,21 @@ readAnnotationWithoutPrefix sandboxed = do int <- many1 digit return $ read int - "enable" -> readName `sepBy` char ',' + "enable" -> plainOrQuoted $ readName `sepBy` char ',' where readName = EnableComment <$> many1 (letter <|> char '-') "source" -> do - filename <- many1 $ noneOf " \n" + filename <- quoted (many1 anyChar) <|> (many1 $ noneOf " \n") return [SourceOverride filename] "source-path" -> do - dirname <- many1 $ noneOf " \n" + dirname <- quoted (many1 anyChar) <|> (many1 $ noneOf " \n") return [SourcePath dirname] "shell" -> do pos <- getPosition - shell <- many1 $ noneOf " \n" + shell <- quoted (many1 anyChar) <|> (many1 $ noneOf " \n") when (isNothing $ shellForExecutable shell) $ parseNoteAt pos ErrorC 1103 "This shell type is unknown. Use e.g. sh or bash." @@ -1049,7 +1060,7 @@ readAnnotationWithoutPrefix sandboxed = do "external-sources" -> do pos <- getPosition - value <- many1 letter + value <- plainOrQuoted $ many1 letter case value of "true" -> if sandboxed From c76b8d9a327812fe8164e9b667650d0c183978d8 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 28 Jul 2022 09:37:23 -0700 Subject: [PATCH 569/763] Let annotations take effect earlier (fixes #2534) --- src/ShellCheck/Checker.hs | 11 ++++++ src/ShellCheck/Parser.hs | 80 +++++++++++++++++++-------------------- 2 files changed, 49 insertions(+), 42 deletions(-) diff --git a/src/ShellCheck/Checker.hs b/src/ShellCheck/Checker.hs index 6518e0d..c8d2c39 100644 --- a/src/ShellCheck/Checker.hs +++ b/src/ShellCheck/Checker.hs @@ -496,6 +496,17 @@ prop_fileCannotEnableExternalSources2 = result == [1144] csCheckSourced = True } +prop_rcCanSuppressEarlyProblems1 = null result + where + result = checkWithRc "disable=1071" emptyCheckSpec { + csScript = "#!/bin/zsh\necho $1" + } + +prop_rcCanSuppressEarlyProblems2 = null result + where + result = checkWithRc "disable=1104" emptyCheckSpec { + csScript = "!/bin/bash\necho 'hello world'" + } return [] runTests = $quickCheckAll diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 4ff45ed..d461fc7 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -38,7 +38,6 @@ import Data.Functor import Data.List (isPrefixOf, isInfixOf, isSuffixOf, partition, sortBy, intercalate, nub, find) import Data.Maybe import Data.Monoid -import Debug.Trace -- STRIP import GHC.Exts (sortWith) import Prelude hiding (readList) import System.IO @@ -458,8 +457,8 @@ called s p = do pos <- getPosition withContext (ContextName pos s) p -withAnnotations anns = - withContext (ContextAnnotation anns) +withAnnotations anns p = + if null anns then p else withContext (ContextAnnotation anns) p readConditionContents single = readCondContents `attempting` lookAhead (do @@ -3258,44 +3257,51 @@ prop_readScript3 = isWarning readScript "#!/bin/bash\necho hello\xA0world" prop_readScript4 = isWarning readScript "#!/usr/bin/perl\nfoo=(" prop_readScript5 = isOk readScript "#!/bin/bash\n#This is an empty script\n\n" prop_readScript6 = isOk readScript "#!/usr/bin/env -S X=FOO bash\n#This is an empty script\n\n" +prop_readScript7 = isOk readScript "#!/bin/zsh\n# shellcheck disable=SC1071\nfor f (a b); echo $f\n" readScriptFile sourced = do start <- startSpan pos <- getPosition - optional $ do - readUtf8Bom - parseProblem ErrorC 1082 - "This file has a UTF-8 BOM. Remove it with: LC_CTYPE=C sed '1s/^...//' < yourscript ." - shebang <- readShebang <|> readEmptyLiteral - let (T_Literal _ shebangString) = shebang - allspacing - annotationStart <- startSpan - fileAnnotations <- readAnnotations rcAnnotations <- if sourced then return [] else do filename <- Mr.asks currentFilename readConfigFile filename - let annotations = fileAnnotations ++ rcAnnotations - annotationId <- endSpan annotationStart - let shellAnnotationSpecified = - any (\x -> case x of ShellOverride {} -> True; _ -> False) annotations - shellFlagSpecified <- isJust <$> Mr.asks shellTypeOverride - let ignoreShebang = shellAnnotationSpecified || shellFlagSpecified - unless ignoreShebang $ - verifyShebang pos (executableFromShebang shebangString) - if ignoreShebang || isValidShell (executableFromShebang shebangString) /= Just False - then do - commands <- withAnnotations annotations readCompoundListOrEmpty - id <- endSpan start - verifyEof - let script = T_Annotation annotationId annotations $ - T_Script id shebang commands - reparseIndices script - else do - many anyChar - id <- endSpan start - return $ T_Script id shebang [] + -- Put the rc annotations on the stack so that one can ignore e.g. SC1084 in .shellcheckrc + withAnnotations rcAnnotations $ do + hasBom <- wasIncluded readUtf8Bom + shebang <- readShebang <|> readEmptyLiteral + let (T_Literal _ shebangString) = shebang + allspacing + annotationStart <- startSpan + fileAnnotations <- readAnnotations + + -- Similarly put the filewide annotations on the stack to allow earlier suppression + withAnnotations fileAnnotations $ do + when (hasBom) $ + parseProblemAt pos ErrorC 1082 + "This file has a UTF-8 BOM. Remove it with: LC_CTYPE=C sed '1s/^...//' < yourscript ." + let annotations = fileAnnotations ++ rcAnnotations + annotationId <- endSpan annotationStart + let shellAnnotationSpecified = + any (\x -> case x of ShellOverride {} -> True; _ -> False) annotations + shellFlagSpecified <- isJust <$> Mr.asks shellTypeOverride + let ignoreShebang = shellAnnotationSpecified || shellFlagSpecified + + unless ignoreShebang $ + verifyShebang pos (executableFromShebang shebangString) + if ignoreShebang || isValidShell (executableFromShebang shebangString) /= Just False + then do + commands <- readCompoundListOrEmpty + id <- endSpan start + verifyEof + let script = T_Annotation annotationId annotations $ + T_Script id shebang commands + reparseIndices script + else do + many anyChar + id <- endSpan start + return $ T_Script id shebang [] where verifyShebang pos s = do @@ -3388,16 +3394,6 @@ parsesCleanly parser string = runIdentity $ do return $ Just . null $ parseNotes userState ++ parseProblems systemState (Left _, _) -> return Nothing --- For printf debugging: print the value of an expression --- Example: return $ dump $ T_Literal id [c] -dump :: Show a => a -> a -- STRIP -dump x = trace (show x) x -- STRIP - --- Like above, but print a specific expression: --- Example: return $ dumps ("Returning: " ++ [c]) $ T_Literal id [c] -dumps :: Show x => x -> a -> a -- STRIP -dumps t = trace (show t) -- STRIP - parseWithNotes parser = do item <- parser state <- getState From 04db46381fe515d6e3e66b7d3c33c60fa0275471 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 28 Jul 2022 19:00:03 -0700 Subject: [PATCH 570/763] Use Data.Map.Strict instead for a ~15% parsing speedup --- src/ShellCheck/Parser.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index d461fc7..aeaf703 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -46,7 +46,7 @@ import Text.Parsec.Error import Text.Parsec.Pos import qualified Control.Monad.Reader as Mr import qualified Control.Monad.State as Ms -import qualified Data.Map as Map +import qualified Data.Map.Strict as Map import Test.QuickCheck.All (quickCheckAll) From 77069f7445681747ff88eaa56b6bd23b596eee99 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Fri, 29 Jul 2022 21:05:33 -0700 Subject: [PATCH 571/763] Store postdominators as Array Node [Node] for a significant win --- src/ShellCheck/CFG.hs | 17 ++++++++++------- src/ShellCheck/CFGAnalysis.hs | 8 ++++---- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/ShellCheck/CFG.hs b/src/ShellCheck/CFG.hs index 39747cf..771e870 100644 --- a/src/ShellCheck/CFG.hs +++ b/src/ShellCheck/CFG.hs @@ -47,6 +47,8 @@ import ShellCheck.Regex import Control.DeepSeq import Control.Monad import Control.Monad.Identity +import Data.Array.Unboxed +import Data.Array.ST import Data.List hiding (map) import Data.Maybe import qualified Data.Map as M @@ -174,10 +176,10 @@ data CFGResult = CFGResult { cfIdToRange :: M.Map Id (Node, Node), -- A set of all nodes belonging to an Id, recursively cfIdToNodes :: M.Map Id (S.Set Node), - -- A map to nodes that the given node postdominates - cfPostDominators :: M.Map Node (S.Set Node) + -- An array (from,to) saying whether 'from' postdominates 'to' + cfPostDominators :: Array Node [Node] } - deriving (Show, Generic, NFData) + deriving (Show) buildGraph :: CFGParameters -> Token -> CFGResult buildGraph params root = @@ -200,7 +202,7 @@ buildGraph params root = cfPostDominators = findPostDominators mainExit $ mkGraph nodes onlyRealEdges } in - deepseq result result + result remapGraph :: M.Map Node Node -> CFW -> CFW remapGraph remap (nodes, edges, mapping, assoc) = @@ -1256,8 +1258,8 @@ findTerminalNodes graph = ufold find [] graph f (IdTagged _ (CFDefineFunction _ id start end):rest) list = f rest (end:list) f (_:rest) list = f rest list -findPostDominators :: Node -> CFGraph -> M.Map Node (S.Set Node) -findPostDominators mainexit graph = asSetMap +findPostDominators :: Node -> CFGraph -> Array Node [Node] +findPostDominators mainexit graph = asArray where inlined = inlineSubshells graph terminals = findTerminalNodes inlined @@ -1265,7 +1267,8 @@ findPostDominators mainexit graph = asSetMap withExitEdges = (incoming ++ map (\c -> (CFEFlow, c)) terminals, mainexit, label, outgoing) `safeUpdate` inlined reversed = grev withExitEdges postDoms = dom reversed mainexit - asSetMap = M.fromList $ map (\(node, list) -> (node, S.fromList list)) postDoms + (_, maxNode) = nodeRange graph + asArray = array (0, maxNode) postDoms return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) diff --git a/src/ShellCheck/CFGAnalysis.hs b/src/ShellCheck/CFGAnalysis.hs index ff88810..e6b1701 100644 --- a/src/ShellCheck/CFGAnalysis.hs +++ b/src/ShellCheck/CFGAnalysis.hs @@ -69,6 +69,7 @@ import Control.Monad import Control.Monad.ST import Control.DeepSeq import Data.List hiding (map) +import Data.Array.Unboxed import Data.STRef import Data.Maybe import qualified Data.Map as M @@ -100,9 +101,9 @@ data CFGAnalysis = CFGAnalysis { graph :: CFGraph, tokenToRange :: M.Map Id (Node, Node), tokenToNodes :: M.Map Id (S.Set Node), - postDominators :: M.Map Node (S.Set Node), + postDominators :: Array Node [Node], nodeToData :: M.Map Node (ProgramState, ProgramState) -} deriving (Show, Generic, NFData) +} deriving (Show) -- The program state we expose externally data ProgramState = ProgramState { @@ -147,8 +148,7 @@ doesPostDominate :: CFGAnalysis -> Id -> Id -> Bool doesPostDominate analysis target base = fromMaybe False $ do (_, baseEnd) <- M.lookup base $ tokenToRange analysis (targetStart, _) <- M.lookup target $ tokenToRange analysis - postDoms <- M.lookup baseEnd $ postDominators analysis - return $ S.member targetStart postDoms + return $ targetStart `elem` (postDominators analysis ! baseEnd) getDataForNode analysis node = M.lookup node $ nodeToData analysis From 0df934514298adadc40651696d5b854784efa0a5 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 2 Aug 2022 11:25:35 -0700 Subject: [PATCH 572/763] Trace numerical status, use for SC2071 (ref #2541) --- src/ShellCheck/Analytics.hs | 21 +++++++-- src/ShellCheck/CFGAnalysis.hs | 84 ++++++++++++++++++++++++++--------- 2 files changed, 82 insertions(+), 23 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index e878dc4..b5bac35 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1167,6 +1167,10 @@ prop_checkNumberComparisons18 = verify checkNumberComparisons "[[ foo -eq 2 ]]" prop_checkNumberComparisons19 = verifyNot checkNumberComparisons "foo=1; [[ foo -eq 2 ]]" prop_checkNumberComparisons20 = verify checkNumberComparisons "[[ 2 -eq / ]]" prop_checkNumberComparisons21 = verify checkNumberComparisons "[[ foo -eq foo ]]" +prop_checkNumberComparisons22 = verify checkNumberComparisons "x=10; [[ $x > $z ]]" +prop_checkNumberComparisons23 = verify checkNumberComparisons "x=0; if [[ -n $def ]]; then x=$def; fi; while [ $x > $z ]; do lol; done" +prop_checkNumberComparisons24 = verify checkNumberComparisons "x=$RANDOM; [ $x > $z ]" +prop_checkNumberComparisons25 = verify checkNumberComparisons "[[ $((n++)) > $x ]]" checkNumberComparisons params (TC_Binary id typ op lhs rhs) = do if isNum lhs || isNum rhs @@ -1242,9 +1246,20 @@ checkNumberComparisons params (TC_Binary id typ op lhs rhs) = do numChar x = isDigit x || x `elem` "+-. " isNum t = - case oversimplify t of - [v] -> all isDigit v - _ -> False + case getWordParts t of + [T_DollarArithmetic {}] -> True + [b@(T_DollarBraced id _ c)] -> + let + str = concat $ oversimplify c + var = getBracedReference str + in fromMaybe False $ do + state <- CF.getIncomingState (cfgAnalysis params) id + value <- Map.lookup var $ CF.variablesInScope state + return $ CF.numericalStatus (CF.variableValue value) >= CF.NumericalStatusMaybe + _ -> + case oversimplify t of + [v] -> all isDigit v + _ -> False isFraction t = case oversimplify t of diff --git a/src/ShellCheck/CFGAnalysis.hs b/src/ShellCheck/CFGAnalysis.hs index e6b1701..634d354 100644 --- a/src/ShellCheck/CFGAnalysis.hs +++ b/src/ShellCheck/CFGAnalysis.hs @@ -54,29 +54,31 @@ module ShellCheck.CFGAnalysis ( ,VariableValue (..) ,VariableProperties ,SpaceStatus (..) + ,NumericalStatus (..) ,getIncomingState ,getOutgoingState ,doesPostDominate ,ShellCheck.CFGAnalysis.runTests -- STRIP ) where -import GHC.Generics (Generic) -import ShellCheck.AST -import ShellCheck.CFG -import qualified ShellCheck.Data as Data -import ShellCheck.Prelude +import Control.DeepSeq import Control.Monad import Control.Monad.ST -import Control.DeepSeq -import Data.List hiding (map) import Data.Array.Unboxed -import Data.STRef -import Data.Maybe -import qualified Data.Map as M -import qualified Data.Set as S +import Data.Char import Data.Graph.Inductive.Graph import Data.Graph.Inductive.Query.DFS +import Data.List hiding (map) +import Data.Maybe +import Data.STRef import Debug.Trace -- STRIP +import GHC.Generics (Generic) +import qualified Data.Map as M +import qualified Data.Set as S +import qualified ShellCheck.Data as Data +import ShellCheck.AST +import ShellCheck.CFG +import ShellCheck.Prelude import Test.QuickCheck @@ -183,16 +185,20 @@ createEnvironmentState = do foldl' (flip ($)) newInternalState $ concat [ addVars Data.internalVariables unknownVariableState, addVars Data.variablesWithoutSpaces spacelessVariableState, - addVars Data.specialIntegerVariables spacelessVariableState + addVars Data.specialIntegerVariables integerVariableState ] where addVars names val = map (\name -> insertGlobal name val) names spacelessVariableState = unknownVariableState { variableValue = VariableValue { literalValue = Nothing, - spaceStatus = SpaceStatusClean + spaceStatus = SpaceStatusClean, + numericalStatus = NumericalStatusUnknown } } + integerVariableState = unknownVariableState { + variableValue = unknownIntegerValue + } modified s = s { sVersion = -1 } @@ -289,7 +295,8 @@ unknownFunctionValue = S.singleton FunctionUnknown -- The information about the value of a single variable data VariableValue = VariableValue { literalValue :: Maybe String, -- TODO: For debugging. Remove me. - spaceStatus :: SpaceStatus + spaceStatus :: SpaceStatus, + numericalStatus :: NumericalStatus } deriving (Show, Eq, Ord, Generic, NFData) @@ -301,6 +308,9 @@ data VariableState = VariableState { -- Whether or not the value needs quoting (has spaces/globs), or we don't know data SpaceStatus = SpaceStatusEmpty | SpaceStatusClean | SpaceStatusDirty deriving (Show, Eq, Ord, Generic, NFData) +-- +-- Whether or not the value needs quoting (has spaces/globs), or we don't know +data NumericalStatus = NumericalStatusUnknown | NumericalStatusEmpty | NumericalStatusMaybe | NumericalStatusDefinitely deriving (Show, Eq, Ord, Generic, NFData) -- The set of possible sets of properties for this variable type VariableProperties = S.Set (S.Set CFVariableProp) @@ -314,12 +324,14 @@ unknownVariableState = VariableState { unknownVariableValue = VariableValue { literalValue = Nothing, - spaceStatus = SpaceStatusDirty + spaceStatus = SpaceStatusDirty, + numericalStatus = NumericalStatusUnknown } emptyVariableValue = unknownVariableValue { literalValue = Just "", - spaceStatus = SpaceStatusEmpty + spaceStatus = SpaceStatusEmpty, + numericalStatus = NumericalStatusEmpty } unsetVariableState = VariableState { @@ -334,7 +346,8 @@ mergeVariableState a b = VariableState { mergeVariableValue a b = VariableValue { literalValue = if literalValue a == literalValue b then literalValue a else Nothing, - spaceStatus = mergeSpaceStatus (spaceStatus a) (spaceStatus b) + spaceStatus = mergeSpaceStatus (spaceStatus a) (spaceStatus b), + numericalStatus = mergeNumericalStatus (numericalStatus a) (numericalStatus b) } mergeSpaceStatus a b = @@ -344,6 +357,16 @@ mergeSpaceStatus a b = (SpaceStatusClean, SpaceStatusClean) -> SpaceStatusClean _ -> SpaceStatusDirty +mergeNumericalStatus a b = + case (a,b) of + (NumericalStatusDefinitely, NumericalStatusDefinitely) -> NumericalStatusDefinitely + (NumericalStatusDefinitely, _) -> NumericalStatusMaybe + (_, NumericalStatusDefinitely) -> NumericalStatusMaybe + (NumericalStatusMaybe, _) -> NumericalStatusMaybe + (_, NumericalStatusMaybe) -> NumericalStatusMaybe + (NumericalStatusEmpty, NumericalStatusEmpty) -> NumericalStatusEmpty + _ -> NumericalStatusUnknown + -- A VersionedMap is a Map that keeps an additional integer version to quickly determine if it has changed. -- * Version -1 means it's unknown (possibly and presumably changed) -- * Version 0 means it's empty @@ -1154,7 +1177,8 @@ appendVariableValue :: VariableValue -> VariableValue -> VariableValue appendVariableValue a b = unknownVariableValue { literalValue = liftM2 (++) (literalValue a) (literalValue b), - spaceStatus = appendSpaceStatus (spaceStatus a) (spaceStatus b) + spaceStatus = appendSpaceStatus (spaceStatus a) (spaceStatus b), + numericalStatus = appendNumericalStatus (numericalStatus a) (numericalStatus b) } appendSpaceStatus a b = @@ -1164,14 +1188,25 @@ appendSpaceStatus a b = (SpaceStatusClean, SpaceStatusClean) -> a _ ->SpaceStatusDirty +appendNumericalStatus a b = + case (a,b) of + (NumericalStatusEmpty, x) -> x + (x, NumericalStatusEmpty) -> x + (NumericalStatusDefinitely, NumericalStatusDefinitely) -> NumericalStatusDefinitely + (NumericalStatusUnknown, _) -> NumericalStatusUnknown + (_, NumericalStatusUnknown) -> NumericalStatusUnknown + _ -> NumericalStatusMaybe + unknownIntegerValue = unknownVariableValue { literalValue = Nothing, - spaceStatus = SpaceStatusClean + spaceStatus = SpaceStatusClean, + numericalStatus = NumericalStatusDefinitely } literalToVariableValue str = unknownVariableValue { literalValue = Just str, - spaceStatus = literalToSpaceStatus str + spaceStatus = literalToSpaceStatus str, + numericalStatus = literalToNumericalStatus str } withoutChanges ctx f = do @@ -1191,6 +1226,15 @@ literalToSpaceStatus str = _ | all (`notElem` " \t\n*?[") str -> SpaceStatusClean _ -> SpaceStatusDirty +-- Get the NumericalStatus for a literal string, i.e. whether it's an integer +literalToNumericalStatus str = + case str of + "" -> NumericalStatusEmpty + '-':rest -> if isNumeric rest then NumericalStatusDefinitely else NumericalStatusUnknown + rest -> if isNumeric rest then NumericalStatusDefinitely else NumericalStatusUnknown + where + isNumeric = all isDigit + type StateMap = M.Map Node (InternalState, InternalState) -- Classic, iterative Data Flow Analysis. See Wikipedia for a description of the process. From 4806719035de364dd2d49f97112dd8b3a255e3b0 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 2 Aug 2022 15:47:59 -0700 Subject: [PATCH 573/763] Handle variable assignments from `read` in CFG --- src/ShellCheck/CFG.hs | 35 ++++++++++++++++++++++++++++++++++- src/ShellCheck/Data.hs | 1 + 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/CFG.hs b/src/ShellCheck/CFG.hs index 771e870..e0c6267 100644 --- a/src/ShellCheck/CFG.hs +++ b/src/ShellCheck/CFG.hs @@ -41,6 +41,7 @@ module ShellCheck.CFG ( import GHC.Generics (Generic) import ShellCheck.AST import ShellCheck.ASTLib +import ShellCheck.Data import ShellCheck.Interface import ShellCheck.Prelude import ShellCheck.Regex @@ -936,6 +937,8 @@ handleCommand cmd vars args literalCmd = do Just "mapfile" -> regularExpansionWithStatus vars args $ handleMapfile args Just "readarray" -> regularExpansionWithStatus vars args $ handleMapfile args + Just "read" -> regularExpansionWithStatus vars args $ handleRead args + Just "DEFINE_boolean" -> regularExpansionWithStatus vars args $ handleDEFINE args Just "DEFINE_float" -> regularExpansionWithStatus vars args $ handleDEFINE args Just "DEFINE_integer" -> regularExpansionWithStatus vars args $ handleDEFINE args @@ -1113,7 +1116,7 @@ handleCommand cmd vars args literalCmd = do in IdTagged id $ CFWriteVariable name CFValueArray getFromArg = do - flags <- getGnuOpts "d:n:O:s:u:C:c:t" args + flags <- getGnuOpts flagsForMapfile args (_, arg) <- lookup "" flags name <- getLiteralString arg return (getId arg, name) @@ -1125,6 +1128,36 @@ handleCommand cmd vars args literalCmd = do guard $ isVariableName name return (getId c, name) + handleRead (cmd:args) = newNodeRange $ CFApplyEffects main + where + main = fromMaybe fallback $ do + flags <- getGnuOpts flagsForRead args + return $ fromMaybe (withFields flags) $ withArray flags + + withArray :: [(String, (Token, Token))] -> Maybe [IdTagged CFEffect] + withArray flags = do + (_, token) <- lookup "a" flags + return $ fromMaybe [] $ do + name <- getLiteralString token + return [ IdTagged (getId token) $ CFWriteVariable name CFValueArray ] + + withFields flags = mapMaybe getAssignment flags + + getAssignment :: (String, (Token, Token)) -> Maybe (IdTagged CFEffect) + getAssignment f = do + ("", (t, _)) <- return f + name <- getLiteralString t + return $ IdTagged (getId t) $ CFWriteVariable name CFValueString + + fallback = + let + names = reverse $ map fromJust $ takeWhile isJust $ map (\c -> sequence (getId c, getLiteralString c)) $ reverse args + namesOrDefault = if null names then [(getId cmd, "REPLY")] else names + hasDashA = any (== "a") $ map fst $ getGenericOpts args + value = if hasDashA then CFValueArray else CFValueString + in + map (\(id, name) -> IdTagged id $ CFWriteVariable name value) namesOrDefault + handleDEFINE (cmd:args) = newNodeRange $ CFApplyEffects $ maybeToList findVar where diff --git a/src/ShellCheck/Data.hs b/src/ShellCheck/Data.hs index fb82ca8..4090922 100644 --- a/src/ShellCheck/Data.hs +++ b/src/ShellCheck/Data.hs @@ -159,5 +159,6 @@ shellForExecutable name = _ -> Nothing flagsForRead = "sreu:n:N:i:p:a:t:" +flagsForMapfile = "d:n:O:s:u:C:c:t" declaringCommands = ["local", "declare", "export", "readonly", "typeset", "let"] From ccab132b385a215a37ce7e6f4d2601454b0437b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lawrence=20Vel=C3=A1zquez?= Date: Tue, 20 Sep 2022 17:36:46 -0400 Subject: [PATCH 574/763] Reflow lists of internal shell variables No functional changes; this just makes the next few commits cleaner. --- src/ShellCheck/Data.hs | 41 +++++++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/src/ShellCheck/Data.hs b/src/ShellCheck/Data.hs index 4090922..3c9013f 100644 --- a/src/ShellCheck/Data.hs +++ b/src/ShellCheck/Data.hs @@ -30,23 +30,26 @@ internalVariables = [ -- Bash "BASH", "BASHOPTS", "BASHPID", "BASH_ALIASES", "BASH_ARGC", - "BASH_ARGV", "BASH_CMDS", "BASH_COMMAND", "BASH_EXECUTION_STRING", - "BASH_LINENO", "BASH_REMATCH", "BASH_SOURCE", "BASH_SUBSHELL", - "BASH_VERSINFO", "BASH_VERSION", "COMP_CWORD", "COMP_KEY", - "COMP_LINE", "COMP_POINT", "COMP_TYPE", "COMP_WORDBREAKS", - "COMP_WORDS", "COPROC", "DIRSTACK", "EUID", "FUNCNAME", "GROUPS", - "HISTCMD", "HOSTNAME", "HOSTTYPE", "LINENO", "MACHTYPE", "MAPFILE", - "OLDPWD", "OPTARG", "OPTIND", "OSTYPE", "PIPESTATUS", "PPID", "PWD", - "RANDOM", "READLINE_LINE", "READLINE_POINT", "REPLY", "SECONDS", - "SHELLOPTS", "SHLVL", "UID", "BASH_ENV", "BASH_XTRACEFD", "CDPATH", - "COLUMNS", "COMPREPLY", "EMACS", "ENV", "FCEDIT", "FIGNORE", + "BASH_ARGV", "BASH_CMDS", "BASH_COMMAND", + "BASH_EXECUTION_STRING", "BASH_LINENO", + "BASH_REMATCH", "BASH_SOURCE", "BASH_SUBSHELL", "BASH_VERSINFO", + "BASH_VERSION", "COMP_CWORD", "COMP_KEY", "COMP_LINE", "COMP_POINT", + "COMP_TYPE", "COMP_WORDBREAKS", "COMP_WORDS", "COPROC", "DIRSTACK", + "EUID", "FUNCNAME", "GROUPS", "HISTCMD", + "HOSTNAME", "HOSTTYPE", "LINENO", "MACHTYPE", "MAPFILE", "OLDPWD", + "OPTARG", "OPTIND", "OSTYPE", "PIPESTATUS", "PPID", "PWD", "RANDOM", + "READLINE_LINE", "READLINE_POINT", + "REPLY", "SECONDS", "SHELLOPTS", "SHLVL", "UID", + "BASH_ENV", "BASH_XTRACEFD", "CDPATH", "COLUMNS", + "COMPREPLY", "EMACS", "ENV", "FCEDIT", "FIGNORE", "FUNCNEST", "GLOBIGNORE", "HISTCONTROL", "HISTFILE", "HISTFILESIZE", "HISTIGNORE", "HISTSIZE", "HISTTIMEFORMAT", "HOME", "HOSTFILE", "IFS", - "IGNOREEOF", "INPUTRC", "LANG", "LC_ALL", "LC_COLLATE", "LC_CTYPE", - "LC_MESSAGES", "LC_MONETARY", "LC_NUMERIC", "LC_TIME", "LINES", "MAIL", - "MAILCHECK", "MAILPATH", "OPTERR", "PATH", "POSIXLY_CORRECT", - "PROMPT_COMMAND", "PROMPT_DIRTRIM", "PS1", "PS2", "PS3", "PS4", "SHELL", - "TIMEFORMAT", "TMOUT", "TMPDIR", "auto_resume", "histchars", "COPROC", + "IGNOREEOF", "INPUTRC", "LANG", "LC_ALL", "LC_COLLATE", + "LC_CTYPE", "LC_MESSAGES", "LC_MONETARY", "LC_NUMERIC", "LC_TIME", + "LINES", "MAIL", "MAILCHECK", "MAILPATH", "OPTERR", "PATH", + "POSIXLY_CORRECT", "PROMPT_COMMAND", "PROMPT_DIRTRIM", "PS1", + "PS2", "PS3", "PS4", "SHELL", "TIMEFORMAT", "TMOUT", "TMPDIR", + "auto_resume", "histchars", "COPROC", -- Other "USER", "TZ", "TERM", "LOGNAME", "LD_LIBRARY_PATH", "LANGUAGE", "DISPLAY", @@ -68,9 +71,11 @@ specialIntegerVariables = [ specialVariablesWithoutSpaces = "-" : specialIntegerVariables variablesWithoutSpaces = specialVariablesWithoutSpaces ++ [ - "BASHPID", "BASH_ARGC", "BASH_LINENO", "BASH_SUBSHELL", "EUID", "LINENO", - "OPTIND", "PPID", "RANDOM", "SECONDS", "SHELLOPTS", "SHLVL", "UID", - "COLUMNS", "HISTFILESIZE", "HISTSIZE", "LINES" + "BASHPID", "BASH_ARGC", "BASH_LINENO", "BASH_SUBSHELL", "EUID", + "LINENO", "OPTIND", "PPID", "RANDOM", + "SECONDS", + "SHELLOPTS", "SHLVL", "UID", "COLUMNS", "HISTFILESIZE", + "HISTSIZE", "LINES" -- shflags , "FLAGS_ERROR", "FLAGS_FALSE", "FLAGS_TRUE" From f28462b01ca634b4649e0a6dd11a0a8a207d5729 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lawrence=20Vel=C3=A1zquez?= Date: Tue, 20 Sep 2022 19:10:39 -0400 Subject: [PATCH 575/763] Remove duplicate "COPROC" from internal vars list --- src/ShellCheck/Data.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Data.hs b/src/ShellCheck/Data.hs index 3c9013f..8c65474 100644 --- a/src/ShellCheck/Data.hs +++ b/src/ShellCheck/Data.hs @@ -49,7 +49,7 @@ internalVariables = [ "LINES", "MAIL", "MAILCHECK", "MAILPATH", "OPTERR", "PATH", "POSIXLY_CORRECT", "PROMPT_COMMAND", "PROMPT_DIRTRIM", "PS1", "PS2", "PS3", "PS4", "SHELL", "TIMEFORMAT", "TMOUT", "TMPDIR", - "auto_resume", "histchars", "COPROC", + "auto_resume", "histchars", -- Other "USER", "TZ", "TERM", "LOGNAME", "LD_LIBRARY_PATH", "LANGUAGE", "DISPLAY", From 966fb3e3dd3b49e64f022478f18f7aa4bad53a27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lawrence=20Vel=C3=A1zquez?= Date: Tue, 20 Sep 2022 19:12:05 -0400 Subject: [PATCH 576/763] Recognize more Bash internal variables - BASH_ARGV0, introduced in Bash 5.0 - BASH_COMPAT, 4.3 - BASH_LOADABLES_PATH, 4.4 - CHILD_MAX, 4.3 - EPOCHREALTIME, 5.0 - EPOCHSECONDS, 5.0 - EXECIGNORE, 4.4 - INSIDE_EMACS, 4.4 - PS0, 4.4 - READLINE_ARGUMENT, 5.2 - READLINE_MARK, 5.1 - SRANDOM, 5.1 Fixes #1780 and #2554. --- src/ShellCheck/Data.hs | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/ShellCheck/Data.hs b/src/ShellCheck/Data.hs index 8c65474..35f314f 100644 --- a/src/ShellCheck/Data.hs +++ b/src/ShellCheck/Data.hs @@ -30,24 +30,24 @@ internalVariables = [ -- Bash "BASH", "BASHOPTS", "BASHPID", "BASH_ALIASES", "BASH_ARGC", - "BASH_ARGV", "BASH_CMDS", "BASH_COMMAND", - "BASH_EXECUTION_STRING", "BASH_LINENO", + "BASH_ARGV", "BASH_ARGV0", "BASH_CMDS", "BASH_COMMAND", + "BASH_EXECUTION_STRING", "BASH_LINENO", "BASH_LOADABLES_PATH", "BASH_REMATCH", "BASH_SOURCE", "BASH_SUBSHELL", "BASH_VERSINFO", "BASH_VERSION", "COMP_CWORD", "COMP_KEY", "COMP_LINE", "COMP_POINT", "COMP_TYPE", "COMP_WORDBREAKS", "COMP_WORDS", "COPROC", "DIRSTACK", - "EUID", "FUNCNAME", "GROUPS", "HISTCMD", + "EPOCHREALTIME", "EPOCHSECONDS", "EUID", "FUNCNAME", "GROUPS", "HISTCMD", "HOSTNAME", "HOSTTYPE", "LINENO", "MACHTYPE", "MAPFILE", "OLDPWD", "OPTARG", "OPTIND", "OSTYPE", "PIPESTATUS", "PPID", "PWD", "RANDOM", - "READLINE_LINE", "READLINE_POINT", - "REPLY", "SECONDS", "SHELLOPTS", "SHLVL", "UID", - "BASH_ENV", "BASH_XTRACEFD", "CDPATH", "COLUMNS", - "COMPREPLY", "EMACS", "ENV", "FCEDIT", "FIGNORE", + "READLINE_ARGUMENT", "READLINE_LINE", "READLINE_MARK", "READLINE_POINT", + "REPLY", "SECONDS", "SHELLOPTS", "SHLVL", "SRANDOM", "UID", "BASH_COMPAT", + "BASH_ENV", "BASH_XTRACEFD", "CDPATH", "CHILD_MAX", "COLUMNS", + "COMPREPLY", "EMACS", "ENV", "EXECIGNORE", "FCEDIT", "FIGNORE", "FUNCNEST", "GLOBIGNORE", "HISTCONTROL", "HISTFILE", "HISTFILESIZE", "HISTIGNORE", "HISTSIZE", "HISTTIMEFORMAT", "HOME", "HOSTFILE", "IFS", - "IGNOREEOF", "INPUTRC", "LANG", "LC_ALL", "LC_COLLATE", + "IGNOREEOF", "INPUTRC", "INSIDE_EMACS", "LANG", "LC_ALL", "LC_COLLATE", "LC_CTYPE", "LC_MESSAGES", "LC_MONETARY", "LC_NUMERIC", "LC_TIME", "LINES", "MAIL", "MAILCHECK", "MAILPATH", "OPTERR", "PATH", - "POSIXLY_CORRECT", "PROMPT_COMMAND", "PROMPT_DIRTRIM", "PS1", + "POSIXLY_CORRECT", "PROMPT_COMMAND", "PROMPT_DIRTRIM", "PS0", "PS1", "PS2", "PS3", "PS4", "SHELL", "TIMEFORMAT", "TMOUT", "TMPDIR", "auto_resume", "histchars", @@ -72,9 +72,9 @@ specialVariablesWithoutSpaces = "-" : specialIntegerVariables variablesWithoutSpaces = specialVariablesWithoutSpaces ++ [ "BASHPID", "BASH_ARGC", "BASH_LINENO", "BASH_SUBSHELL", "EUID", - "LINENO", "OPTIND", "PPID", "RANDOM", - "SECONDS", - "SHELLOPTS", "SHLVL", "UID", "COLUMNS", "HISTFILESIZE", + "EPOCHREALTIME", "EPOCHSECONDS", "LINENO", "OPTIND", "PPID", "RANDOM", + "READLINE_ARGUMENT", "READLINE_MARK", "SECONDS", + "SHELLOPTS", "SHLVL", "SRANDOM", "UID", "COLUMNS", "HISTFILESIZE", "HISTSIZE", "LINES" -- shflags From 0845b8118352c929d414231af23d302cb748178d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lawrence=20Vel=C3=A1zquez?= Date: Tue, 20 Sep 2022 20:00:23 -0400 Subject: [PATCH 577/763] Add READLINE_POINT to list of variables without spaces --- src/ShellCheck/Data.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Data.hs b/src/ShellCheck/Data.hs index 35f314f..7d6f5b4 100644 --- a/src/ShellCheck/Data.hs +++ b/src/ShellCheck/Data.hs @@ -73,7 +73,7 @@ specialVariablesWithoutSpaces = "-" : specialIntegerVariables variablesWithoutSpaces = specialVariablesWithoutSpaces ++ [ "BASHPID", "BASH_ARGC", "BASH_LINENO", "BASH_SUBSHELL", "EUID", "EPOCHREALTIME", "EPOCHSECONDS", "LINENO", "OPTIND", "PPID", "RANDOM", - "READLINE_ARGUMENT", "READLINE_MARK", "SECONDS", + "READLINE_ARGUMENT", "READLINE_MARK", "READLINE_POINT", "SECONDS", "SHELLOPTS", "SHLVL", "SRANDOM", "UID", "COLUMNS", "HISTFILESIZE", "HISTSIZE", "LINES" From fcc473e27fffc2dfe404c24b30f2cbb7e897ace8 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 21 Sep 2022 18:11:18 -0700 Subject: [PATCH 578/763] Include inherited env for DFA of leftover functions (fixes #2560) --- src/ShellCheck/Analytics.hs | 2 ++ src/ShellCheck/CFGAnalysis.hs | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index b5bac35..e9ea36a 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2099,6 +2099,8 @@ prop_checkSpacefulnessCfg61 = verify checkSpacefulnessCfg "declare -x X; y=foo$X prop_checkSpacefulnessCfg62 = verifyNot checkSpacefulnessCfg "f() { declare -x X; y=foo$X; echo $y; }" prop_checkSpacefulnessCfg63 = verify checkSpacefulnessCfg "f && declare -i s; s='x + y'; echo $s" prop_checkSpacefulnessCfg64 = verifyNot checkSpacefulnessCfg "declare -i s; s='x + y'; x=$s; echo $x" +prop_checkSpacefulnessCfg65 = verifyNot checkSpacefulnessCfg "f() { s=$?; echo $s; }; f" +prop_checkSpacefulnessCfg66 = verifyNot checkSpacefulnessCfg "f() { s=$?; echo $s; }" checkSpacefulnessCfg = checkSpacefulnessCfg' True checkVerboseSpacefulnessCfg = checkSpacefulnessCfg' False diff --git a/src/ShellCheck/CFGAnalysis.hs b/src/ShellCheck/CFGAnalysis.hs index 634d354..7b270a8 100644 --- a/src/ShellCheck/CFGAnalysis.hs +++ b/src/ShellCheck/CFGAnalysis.hs @@ -1300,8 +1300,7 @@ dataflow ctx entry = do outgoing = map snd outgoingL isRegular = ((== CFEFlow) . fst) -runRoot ctx entry exit = do - let env = createEnvironmentState +runRoot ctx env entry exit = do writeSTRef (cInput ctx) $ env writeSTRef (cOutput ctx) $ env writeSTRef (cNode ctx) $ entry @@ -1321,9 +1320,10 @@ analyzeControlFlow params t = runST $ f cfg entry exit where f cfg entry exit = do + let env = createEnvironmentState ctx <- newCtx $ cfGraph cfg -- Do a dataflow analysis starting on the root node - exitState <- runRoot ctx entry exit + exitState <- runRoot ctx env entry exit -- All nodes we've touched invocations <- readSTRef $ cInvocations ctx @@ -1336,7 +1336,7 @@ analyzeControlFlow params t = let uninvoked = M.difference declaredFunctions invokedNodes let stragglerInput = - exitState { + (env `patchState` exitState) { -- We don't want `die() { exit $?; }; echo "Sourced"` to assume $? is always echo sExitCodes = Nothing } From 581981ba7696e642ce4ffa57d6cc2a585ad631d5 Mon Sep 17 00:00:00 2001 From: Christian Nassif-Haynes Date: Sat, 24 Sep 2022 07:20:48 +1000 Subject: [PATCH 579/763] Suppress SC2311 with `set -o posix` --- src/ShellCheck/Analytics.hs | 1 + src/ShellCheck/AnalyzerLib.hs | 21 +++++++++------------ 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index e9ea36a..9d924f6 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -4686,6 +4686,7 @@ prop_checkSetESuppressed15 = verifyTree checkSetESuppressed "set -e; f(){ :; prop_checkSetESuppressed16 = verifyTree checkSetESuppressed "set -e; f(){ :; }; until set -e; f; do :; done" prop_checkSetESuppressed17 = verifyNotTree checkSetESuppressed "set -e; f(){ :; }; g(){ :; }; g f" prop_checkSetESuppressed18 = verifyNotTree checkSetESuppressed "set -e; shopt -s inherit_errexit; f(){ :; }; x=$(f)" +prop_checkSetESuppressed19 = verifyNotTree checkSetESuppressed "set -e; set -o posix; f(){ :; }; x=$(f)" checkSetESuppressed params t = if hasSetE params then runNodeAnalysis checkNode params t else [] where diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 88da89e..444c751 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -203,22 +203,22 @@ makeParameters spec = params hasSetE = containsSetE root, hasLastpipe = case shellType params of - Bash -> containsLastpipe root + Bash -> isOptionSet "lastpipe" root Dash -> False Sh -> False Ksh -> True, hasInheritErrexit = case shellType params of - Bash -> containsInheritErrexit root + Bash -> isOptionSet "inherit_errexit" root Dash -> True Sh -> True Ksh -> False, hasPipefail = case shellType params of - Bash -> containsPipefail root + Bash -> isOptionSet "pipefail" root Dash -> True Sh -> True - Ksh -> containsPipefail root, + Ksh -> isOptionSet "pipefail" root, shellTypeSpecified = isJust (asShellType spec) || isJust (asFallbackShell spec), idMap = getTokenMap root, parentMap = getParentTree root, @@ -247,13 +247,14 @@ containsSetE root = isNothing $ doAnalysis (guard . not . isSetE) root _ -> False re = mkRegex "[[:space:]]-[^-]*e" -containsPipefail root = isNothing $ doAnalysis (guard . not . isPipefail) root + +containsSetOption opt root = isNothing $ doAnalysis (guard . not . isPipefail) root where isPipefail t = case t of T_SimpleCommand {} -> t `isUnqualifiedCommand` "set" && - ("pipefail" `elem` oversimplify t || + (opt `elem` oversimplify t || "o" `elem` map snd (getAllFlags t)) _ -> False @@ -267,12 +268,8 @@ containsShopt shopt root = (shopt `elem` oversimplify t) _ -> False --- Does this script mention 'shopt -s inherit_errexit' anywhere? -containsInheritErrexit = containsShopt "inherit_errexit" - --- Does this script mention 'shopt -s lastpipe' anywhere? --- Also used as a hack. -containsLastpipe = containsShopt "lastpipe" +-- Does this script mention 'shopt -s $opt' or 'set -o $opt' anywhere? +isOptionSet opt root = containsShopt opt root || containsSetOption opt root prop_determineShell0 = determineShellTest "#!/bin/sh" == Sh From ef5f9a7af5e2f8287c535a45bba352ea193aab4b Mon Sep 17 00:00:00 2001 From: Christian Nassif-Haynes Date: Sun, 25 Sep 2022 03:04:20 +1000 Subject: [PATCH 580/763] Add `mapfile` to harmless commands for SC2094 --- src/ShellCheck/Analytics.hs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index e9ea36a..7237462 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -807,6 +807,7 @@ prop_checkRedirectToSame6 = verifyNot checkRedirectToSame "echo foo > foo" prop_checkRedirectToSame7 = verifyNot checkRedirectToSame "sed 's/foo/bar/g' file | sponge file" prop_checkRedirectToSame8 = verifyNot checkRedirectToSame "while read -r line; do _=\"$fname\"; done <\"$fname\"" prop_checkRedirectToSame9 = verifyNot checkRedirectToSame "while read -r line; do cat < \"$fname\"; done <\"$fname\"" +prop_checkRedirectToSame10 = verifyNot checkRedirectToSame "mapfile -t foo (mapM_ (\x -> doAnalysis (checkOccurrences x) l) (getAllRedirs list))) list where @@ -852,7 +853,7 @@ checkRedirectToSame params s@(T_Pipeline _ _ list) = isHarmlessCommand arg = fromMaybe False $ do cmd <- getClosestCommand (parentMap params) arg name <- getCommandBasename cmd - return $ name `elem` ["echo", "printf", "sponge"] + return $ name `elem` ["echo", "mapfile", "printf", "sponge"] containsAssignment arg = fromMaybe False $ do cmd <- getClosestCommand (parentMap params) arg return $ isAssignment cmd From 128351f5ef002be2ca3d2b2e3e2859c2c6c84e9a Mon Sep 17 00:00:00 2001 From: Peter Oliver Date: Fri, 7 Oct 2022 17:02:31 +0100 Subject: [PATCH 581/763] Permit colon after exec MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ShellCheck throws warning SC2093 when a script contains commands that could never be executed because they are after an `exec`. Command `:` does nothing, so add it to the list of commands that don’t trigger this warning. --- src/ShellCheck/Analytics.hs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index aa99934..ba2379d 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1876,6 +1876,7 @@ prop_checkSpuriousExec7 = verifyNot checkSpuriousExec "exec file; echo failed; e prop_checkSpuriousExec8 = verifyNot checkSpuriousExec "exec {origout}>&1- >tmp.log 2>&1; bar" prop_checkSpuriousExec9 = verify checkSpuriousExec "for file in rc.d/*; do exec \"$file\"; done" prop_checkSpuriousExec10 = verifyNot checkSpuriousExec "exec file; r=$?; printf >&2 'failed\n'; return $r" +prop_checkSpuriousExec11 = verifyNot checkSpuriousExec "exec file; :" checkSpuriousExec _ = doLists where doLists (T_Script _ _ cmds) = doList cmds False @@ -1891,7 +1892,7 @@ checkSpuriousExec _ = doLists stripCleanup = reverse . dropWhile cleanup . reverse cleanup (T_Pipeline _ _ [cmd]) = - isCommandMatch cmd (`elem` ["echo", "exit", "printf", "return"]) + isCommandMatch cmd (`elem` [":", "echo", "exit", "printf", "return"]) || isAssignment cmd cleanup _ = False From 43aca62ca7ffa84a406623d81fd67758b577bfa5 Mon Sep 17 00:00:00 2001 From: Christian Nassif-Haynes Date: Sun, 9 Oct 2022 07:59:05 +1100 Subject: [PATCH 582/763] Fix false positive for SC2312 when using `time` --- src/ShellCheck/Analytics.hs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index aa99934..bf77c9c 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -4765,8 +4765,12 @@ prop_checkExtraMaskedReturns32 = verifyNotTree checkExtraMaskedReturns "false < prop_checkExtraMaskedReturns33 = verifyNotTree checkExtraMaskedReturns "{ false || true; } | true" prop_checkExtraMaskedReturns34 = verifyNotTree checkExtraMaskedReturns "{ false || :; } | true" prop_checkExtraMaskedReturns35 = verifyTree checkExtraMaskedReturns "f() { local -r x=$(false); }" +prop_checkExtraMaskedReturns36 = verifyNotTree checkExtraMaskedReturns "time false" +prop_checkExtraMaskedReturns37 = verifyNotTree checkExtraMaskedReturns "time $(time false)" +prop_checkExtraMaskedReturns38 = verifyTree checkExtraMaskedReturns "x=$(time time time false) time $(time false)" -checkExtraMaskedReturns params t = runNodeAnalysis findMaskingNodes params t +checkExtraMaskedReturns params t = + runNodeAnalysis findMaskingNodes params (removeTransparentCommands t) where findMaskingNodes _ (T_Arithmetic _ list) = findMaskedNodesInList [list] findMaskingNodes _ (T_Array _ list) = findMaskedNodesInList $ allButLastSimpleCommands list @@ -4799,6 +4803,13 @@ checkExtraMaskedReturns params t = runNodeAnalysis findMaskingNodes params t where simpleCommands = filter containsSimpleCommand cmds + removeTransparentCommands t = + doTransform go t + where + go cmd@(T_SimpleCommand id assigns (_:args)) | isTransparentCommand cmd + = T_SimpleCommand id assigns args + go t = t + inform t = info (getId t) 2312 ("Consider invoking this command " ++ "separately to avoid masking its return value (or use '|| true' " ++ "to ignore).") @@ -4831,6 +4842,10 @@ checkExtraMaskedReturns params t = runNodeAnalysis findMaskingNodes params t ,"shopt" ] + isTransparentCommand t = fromMaybe False $ do + basename <- getCommandBasename t + return $ basename == "time" + parentChildPairs t = go $ parents params t where go (child:parent:rest) = (parent, child):go (parent:rest) From 81c2ecaccb47ced29e9e596f0d53b3068eff6811 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 11 Oct 2022 19:40:29 -0700 Subject: [PATCH 583/763] Remove true/false from SC2216/SC2217 (fixes #2603) --- src/ShellCheck/Data.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Data.hs b/src/ShellCheck/Data.hs index 7d6f5b4..550ff87 100644 --- a/src/ShellCheck/Data.hs +++ b/src/ShellCheck/Data.hs @@ -121,10 +121,10 @@ commonCommands = [ nonReadingCommands = [ "alias", "basename", "bg", "cal", "cd", "chgrp", "chmod", "chown", - "cp", "du", "echo", "export", "false", "fg", "fuser", "getconf", + "cp", "du", "echo", "export", "fg", "fuser", "getconf", "getopt", "getopts", "ipcrm", "ipcs", "jobs", "kill", "ln", "ls", "locale", "mv", "printf", "ps", "pwd", "renice", "rm", "rmdir", - "set", "sleep", "touch", "trap", "true", "ulimit", "unalias", "uname" + "set", "sleep", "touch", "trap", "ulimit", "unalias", "uname" ] sampleWords = [ From fa7943ac0e79dc3ac94bede40c55b8f407a699d6 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 11 Oct 2022 20:10:34 -0700 Subject: [PATCH 584/763] Revert "Add employer mandated disclaimer" This reverts commit 5202072a3439935fbc5a9b92fe66833633f63437. --- LICENSE | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/LICENSE b/LICENSE index 0df6056..f288702 100644 --- a/LICENSE +++ b/LICENSE @@ -1,13 +1,3 @@ -Employer mandated disclaimer: - - I am providing code in the repository to you under an open source license. - Because this is my personal repository, the license you receive to my code is - from me and other individual contributors, and not my employer (Facebook). - - - Vidar "koala_man" Holen - ----- - GNU GENERAL PUBLIC LICENSE Version 3, 29 June 2007 From a524929b6920ed4950b5bf5a97c57a40dd9b09f5 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 12 Oct 2022 20:21:48 -0700 Subject: [PATCH 585/763] Remove outdated test --- src/ShellCheck/Analytics.hs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index bf77c9c..e8bc4b9 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -3575,7 +3575,6 @@ prop_checkPipeToNowhere4 = verify checkPipeToNowhere "printf 'Lol' << eof\nlol\n prop_checkPipeToNowhere5 = verifyNot checkPipeToNowhere "echo foo | xargs du" prop_checkPipeToNowhere6 = verifyNot checkPipeToNowhere "ls | echo $(cat)" prop_checkPipeToNowhere7 = verifyNot checkPipeToNowhere "echo foo | var=$(cat) ls" -prop_checkPipeToNowhere8 = verify checkPipeToNowhere "foo | true" prop_checkPipeToNowhere9 = verifyNot checkPipeToNowhere "mv -i f . < /dev/stdin" prop_checkPipeToNowhere10 = verify checkPipeToNowhere "ls > file | grep foo" prop_checkPipeToNowhere11 = verify checkPipeToNowhere "ls | grep foo < file" From 14056a7f3a5917cba81582b33624948e83d7a50d Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 12 Oct 2022 20:20:59 -0700 Subject: [PATCH 586/763] Don't suggest pgrep for `ps -p .. | grep` (fixes #2597) --- src/ShellCheck/Analytics.hs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index e8bc4b9..77c527f 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -550,6 +550,7 @@ prop_checkPipePitfalls19 = verifyNot checkPipePitfalls "foo | grep -A2 bar | wc prop_checkPipePitfalls20 = verifyNot checkPipePitfalls "foo | grep -B999 bar | wc -l" prop_checkPipePitfalls21 = verifyNot checkPipePitfalls "foo | grep --after-context 999 bar | wc -l" prop_checkPipePitfalls22 = verifyNot checkPipePitfalls "foo | grep -B 1 --after-context 999 bar | wc -l" +prop_checkPipePitfalls23 = verifyNot checkPipePitfalls "ps -o pid,args -p $(pgrep java) | grep -F net.shellcheck.Test" checkPipePitfalls _ (T_Pipeline id _ commands) = do for ["find", "xargs"] $ \(find:xargs:_) -> @@ -563,8 +564,15 @@ checkPipePitfalls _ (T_Pipeline id _ commands) = do ]) $ warn (getId find) 2038 "Use -print0/-0 or -exec + to allow for non-alphanumeric filenames." - for' ["ps", "grep"] $ - \x -> info x 2009 "Consider using pgrep instead of grepping ps output." + for ["ps", "grep"] $ + \(ps:grep:_) -> + let + psFlags = maybe [] (map snd . getAllFlags) $ getCommand ps + in + -- There are many ways to specify a pid: 1, -1, p 1, wup 1, -q 1, -p 1, --pid 1. + -- For simplicity we only deal with the most canonical looking flags: + unless (any (`elem` ["p", "pid", "q", "quick-pid"]) psFlags) $ + info (getId ps) 2009 "Consider using pgrep instead of grepping ps output." for ["grep", "wc"] $ \(grep:wc:_) -> @@ -782,6 +790,7 @@ prop_checkUnquotedExpansions7 = verifyNot checkUnquotedExpansions "cat << foo\n$ prop_checkUnquotedExpansions8 = verifyNot checkUnquotedExpansions "set -- $(seq 1 4)" prop_checkUnquotedExpansions9 = verifyNot checkUnquotedExpansions "echo foo `# inline comment`" prop_checkUnquotedExpansions10 = verify checkUnquotedExpansions "#!/bin/sh\nexport var=$(val)" +prop_checkUnquotedExpansions11 = verifyNot checkUnquotedExpansions "ps -p $(pgrep foo)" checkUnquotedExpansions params = check where @@ -795,7 +804,7 @@ checkUnquotedExpansions params = warn (getId t) 2046 "Quote this to prevent word splitting." shouldBeSplit t = - getCommandNameFromExpansion t == Just "seq" + getCommandNameFromExpansion t `elem` [Just "seq", Just "pgrep"] prop_checkRedirectToSame = verify checkRedirectToSame "cat foo > foo" From d9c9e60fb0064a45188dd93993ad86253ac5a0b3 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 13 Oct 2022 19:46:15 -0700 Subject: [PATCH 587/763] Allow arbitrary bats @test names (fixes #2587) --- src/ShellCheck/AST.hs | 2 +- src/ShellCheck/Parser.hs | 19 ++++++++++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/AST.hs b/src/ShellCheck/AST.hs index ca5007a..5c20416 100644 --- a/src/ShellCheck/AST.hs +++ b/src/ShellCheck/AST.hs @@ -142,7 +142,7 @@ data InnerToken t = | Inner_T_CoProcBody t | Inner_T_Include t | Inner_T_SourceCommand t t - | Inner_T_BatsTest t t + | Inner_T_BatsTest String t deriving (Show, Eq, Functor, Foldable, Traversable) data Annotation = diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index aeaf703..969f4b7 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -2500,16 +2500,29 @@ readBraceGroup = called "brace group" $ do spacing return $ T_BraceGroup id list -prop_readBatsTest = isOk readBatsTest "@test 'can parse' {\n true\n}" +prop_readBatsTest1 = isOk readBatsTest "@test 'can parse' {\n true\n}" +prop_readBatsTest2 = isOk readBatsTest "@test random text !(@*$Y&! {\n true\n}" +prop_readBatsTest3 = isOk readBatsTest "@test foo { bar { baz {\n true\n}" +prop_readBatsTest4 = isNotOk readBatsTest "@test foo \n{\n true\n}" readBatsTest = called "bats @test" $ do start <- startSpan - try $ string "@test" + try $ string "@test " spacing - name <- readNormalWord + name <- readBatsName spacing test <- readBraceGroup id <- endSpan start return $ T_BatsTest id name test + where + readBatsName = do + line <- try . lookAhead $ many1 $ noneOf "\n" + let name = reverse $ f $ reverse line + string name + + -- We want everything before the last " {" in a string, so we find everything after "{ " in its reverse + f ('{':' ':rest) = dropWhile isSpace rest + f (a:rest) = f rest + f [] = "" prop_readWhileClause = isOk readWhileClause "while [[ -e foo ]]; do sleep 1; done" readWhileClause = called "while loop" $ do From b770984dfcfb90d74b2ca7c1ad11cd35bb45d45e Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 13 Oct 2022 21:04:38 -0700 Subject: [PATCH 588/763] Try to parse the inside of traps (fixes #2584) --- src/ShellCheck/Parser.hs | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 969f4b7..837735a 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -2112,6 +2112,7 @@ prop_readSimpleCommand11 = isOk readSimpleCommand "/\\* foo" prop_readSimpleCommand12 = isWarning readSimpleCommand "elsif foo" prop_readSimpleCommand13 = isWarning readSimpleCommand "ElseIf foo" prop_readSimpleCommand14 = isWarning readSimpleCommand "elseif[$i==2]" +prop_readSimpleCommand15 = isWarning readSimpleCommand "trap 'foo\"bar' INT" readSimpleCommand = called "simple command" $ do prefix <- option [] readCmdPrefix skipAnnotationAndWarn @@ -2141,9 +2142,12 @@ readSimpleCommand = called "simple command" $ do id2 <- getNewIdFor id1 let result = makeSimpleCommand id1 id2 prefix [cmd] suffix - if isCommand ["source", "."] cmd - then readSource result - else return result + case () of + _ | isCommand ["source", "."] cmd -> readSource result + _ | isCommand ["trap"] cmd -> do + syntaxCheckTrap result + return result + _ -> return result where isCommand strings (T_NormalWord _ [T_Literal _ s]) = s `elem` strings isCommand _ _ = False @@ -2163,6 +2167,17 @@ readSimpleCommand = called "simple command" $ do parseProblemAtId (getId cmd) ErrorC 1131 "Use 'elif' to start another branch." _ -> return () + syntaxCheckTrap cmd = + case cmd of + (T_Redirecting _ _ (T_SimpleCommand _ _ (cmd:arg:_))) -> checkArg arg (getLiteralString arg) + _ -> return () + where + checkArg _ Nothing = return () + checkArg arg (Just ('-':_)) = return () + checkArg arg (Just str) = do + (start,end) <- getSpanForId (getId arg) + subParse start (tryWithErrors (readCompoundListOrEmpty >> verifyEof) <|> return ()) str + commentWarning id = parseProblemAtId id ErrorC 1127 "Was this intended as a comment? Use # in sh." From 86e2b76730bef5ec509828d138c67d6b49b89b7f Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 29 Oct 2022 12:50:07 -0700 Subject: [PATCH 589/763] Improve SC1059 error message --- src/ShellCheck/Parser.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 837735a..dd0f0f0 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -2566,7 +2566,7 @@ readDoGroup kwId = do parseProblem ErrorC 1058 "Expected 'do'." return "Expected 'do'" - acceptButWarn g_Semi ErrorC 1059 "No semicolons directly after 'do'." + acceptButWarn g_Semi ErrorC 1059 "Semicolon is not allowed directly after 'do'. You can just delete it." allspacing optional (do From 84d8530f14e35ff3fb7688fb11935b60b02deaf0 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 29 Oct 2022 12:50:37 -0700 Subject: [PATCH 590/763] Add SVG logo --- doc/shellcheck_logo.svg | 294 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 294 insertions(+) create mode 100644 doc/shellcheck_logo.svg diff --git a/doc/shellcheck_logo.svg b/doc/shellcheck_logo.svg new file mode 100644 index 0000000..836aa63 --- /dev/null +++ b/doc/shellcheck_logo.svg @@ -0,0 +1,294 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 3342902d9a354c71ba72a939af82d535f043d079 Mon Sep 17 00:00:00 2001 From: ArenM Date: Thu, 17 Nov 2022 18:06:10 -0500 Subject: [PATCH 591/763] Warn about 'read' without a variable in POSIX sh Dash throws an error if the read command isn't supplied a variable name. --- src/ShellCheck/Checks/ShellSupport.hs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 30a19b9..eda6882 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -184,6 +184,10 @@ prop_checkBashisms96 = verifyNot checkBashisms "#!/bin/dash\necho $_" prop_checkBashisms97 = verify checkBashisms "#!/bin/sh\necho ${var,}" prop_checkBashisms98 = verify checkBashisms "#!/bin/sh\necho ${var^^}" prop_checkBashisms99 = verify checkBashisms "#!/bin/dash\necho [^f]oo" +prop_checkBashisms100 = verify checkBashisms "read -r" +prop_checkBashisms101 = verify checkBashisms "read" +prop_checkBashisms102 = verifyNot checkBashisms "read -r foo" +prop_checkBashisms103 = verifyNot checkBashisms "read foo" checkBashisms = ForShell [Sh, Dash] $ \t -> do params <- ask kludge params t @@ -284,6 +288,13 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do argString = concat $ oversimplify arg flagRegex = mkRegex "^-[eEsn]+$" + bashism t@(T_SimpleCommand _ _ (cmd:args)) + | t `isCommand` "read" && length (onlyNames args) == 0 = + warnMsg (getId cmd) 3061 "read without a variable is" + where + notFlag arg = head (concat $ oversimplify arg) /= '-' + onlyNames = filter (notFlag) + bashism t@(T_SimpleCommand _ _ (cmd:arg:_)) | getLiteralString cmd == Just "exec" && "-" `isPrefixOf` concat (oversimplify arg) = warnMsg (getId arg) 3038 "exec flags are" From 2a16a4e8c18745887ca33c545854bdcb3097fede Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 10 Dec 2022 10:46:49 -0800 Subject: [PATCH 592/763] Add missing imports for later GHC versions --- ShellCheck.cabal | 9 ++++++--- shellcheck.hs | 2 ++ src/ShellCheck/AnalyzerLib.hs | 1 + src/ShellCheck/Fixer.hs | 1 + 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/ShellCheck.cabal b/ShellCheck.cabal index b22b5c8..abb32d0 100644 --- a/ShellCheck.cabal +++ b/ShellCheck.cabal @@ -54,11 +54,12 @@ library Diff >= 0.2.0, directory >= 1.2.3.0, fgl, - mtl >= 2.2.1, filepath, + mtl >= 2.2.1, parsec, - regex-tdfa, QuickCheck >= 2.7.4, + regex-tdfa, + transformers, -- When cabal supports it, move this to setup-depends: process exposed-modules: @@ -112,6 +113,7 @@ executable shellcheck parsec >= 3.0, QuickCheck >= 2.7.4, regex-tdfa, + transformers, ShellCheck default-language: Haskell98 main-is: shellcheck.hs @@ -128,11 +130,12 @@ test-suite test-shellcheck Diff >= 0.2.0, directory >= 1.2.3.0, fgl, - mtl >= 2.2.1, filepath, + mtl >= 2.2.1, parsec, QuickCheck >= 2.7.4, regex-tdfa, + transformers, ShellCheck default-language: Haskell98 main-is: test/shellcheck.hs diff --git a/shellcheck.hs b/shellcheck.hs index a525251..4e8a155 100644 --- a/shellcheck.hs +++ b/shellcheck.hs @@ -34,6 +34,8 @@ import qualified ShellCheck.Formatter.Quiet import Control.Exception import Control.Monad +import Control.Monad.IO.Class +import Control.Monad.Trans.Class import Control.Monad.Except import Data.Bits import Data.Char diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 444c751..47ea91d 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -32,6 +32,7 @@ import ShellCheck.Regex import Control.Arrow (first) import Control.DeepSeq +import Control.Monad import Control.Monad.Identity import Control.Monad.RWS import Control.Monad.State diff --git a/src/ShellCheck/Fixer.hs b/src/ShellCheck/Fixer.hs index 358dec9..0d3c8f4 100644 --- a/src/ShellCheck/Fixer.hs +++ b/src/ShellCheck/Fixer.hs @@ -23,6 +23,7 @@ module ShellCheck.Fixer (applyFix, removeTabStops, mapPositions, Ranged(..), run import ShellCheck.Interface import ShellCheck.Prelude +import Control.Monad import Control.Monad.State import Data.Array import Data.List From 495e34d10179715e8a675ca3e721b47757c9dc0f Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 11 Dec 2022 14:18:47 -0800 Subject: [PATCH 593/763] Add missing Semigroup import for older GHC --- src/ShellCheck/Prelude.hs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/ShellCheck/Prelude.hs b/src/ShellCheck/Prelude.hs index 7e9011b..7610c46 100644 --- a/src/ShellCheck/Prelude.hs +++ b/src/ShellCheck/Prelude.hs @@ -21,6 +21,9 @@ -- Generic basic utility functions module ShellCheck.Prelude where +import Data.Semigroup + + -- Get element 0 or a default. Like `head` but safe. headOrDefault _ (a:_) = a headOrDefault def _ = def From 74b1745a1998c5b0a203a5d3c67dade7997ad1b2 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 11 Dec 2022 14:48:00 -0800 Subject: [PATCH 594/763] Fix compiler error on some GHC versions Fixes the following error: src/ShellCheck/CFGAnalysis.hs:1394:40: error: * Couldn't match expected type `[S.Set a]' with actual type `M.Map String FunctionValue' * In the second argument of `($)', namely `mapStorage $ sFunctionTargets state' In the expression: S.unions $ mapStorage $ sFunctionTargets state In an equation for `declaredFuncs': declaredFuncs = S.unions $ mapStorage $ sFunctionTargets state * Relevant bindings include declaredFuncs :: S.Set a (bound at src/ShellCheck/CFGAnalysis.hs:1394:13) --- src/ShellCheck/CFGAnalysis.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/CFGAnalysis.hs b/src/ShellCheck/CFGAnalysis.hs index 7b270a8..cac913e 100644 --- a/src/ShellCheck/CFGAnalysis.hs +++ b/src/ShellCheck/CFGAnalysis.hs @@ -1391,7 +1391,7 @@ analyzeControlFlow params t = getFunctionTargets :: InternalState -> M.Map Node FunctionDefinition getFunctionTargets state = let - declaredFuncs = S.unions $ mapStorage $ sFunctionTargets state + declaredFuncs = S.unions $ M.elems $ mapStorage $ sFunctionTargets state getFunc d = case d of FunctionDefinition _ entry _ -> Just (entry, d) From 3cae6cd6abe16b18a1c95598561d5afca1135cec Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 11 Dec 2022 15:05:33 -0800 Subject: [PATCH 595/763] Allow building on deepseq < 1.4.2.0 --- src/ShellCheck/CFGAnalysis.hs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/ShellCheck/CFGAnalysis.hs b/src/ShellCheck/CFGAnalysis.hs index cac913e..4e36cf5 100644 --- a/src/ShellCheck/CFGAnalysis.hs +++ b/src/ShellCheck/CFGAnalysis.hs @@ -20,6 +20,7 @@ {-# LANGUAGE TemplateHaskell #-} {-# LANGUAGE RankNTypes #-} {-# LANGUAGE DeriveAnyClass, DeriveGeneric #-} +{-# LANGUAGE CPP #-} {- Data Flow Analysis on a Control Flow Graph. @@ -433,6 +434,13 @@ data StackEntry s = StackEntry { } deriving (Eq, Generic, NFData) +#if MIN_VERSION_deepseq(1,4,2) +-- Our deepseq already has a STRef instance +#else +-- Older deepseq (for GHC < 8) lacks this instance +instance NFData (STRef s a) where + rnf = (`seq` ()) +#endif -- Overwrite a base state with the contents of a diff state -- This is unrelated to join/merge. From 985ca2530d475f6bf93fa81d31cc220fccddeea8 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 11 Dec 2022 16:34:29 -0800 Subject: [PATCH 596/763] Add Docker testing for older and newer Ubuntu versions --- test/buildtest | 3 ++- test/distrotest | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/test/buildtest b/test/buildtest index 1d194fc..469539b 100755 --- a/test/buildtest +++ b/test/buildtest @@ -22,7 +22,8 @@ fi cabal install --dependencies-only --enable-tests "${flags[@]}" || cabal install --dependencies-only "${flags[@]}" || - die "can't install dependencies" + cabal install --dependencies-only --max-backjumps -1 "${flags[@]}" || + die "can't install dependencies" cabal configure --enable-tests "${flags[@]}" || die "configure failed" cabal build || diff --git a/test/distrotest b/test/distrotest index 464768c..e1711ea 100755 --- a/test/distrotest +++ b/test/distrotest @@ -67,7 +67,10 @@ fedora:latest dnf install -y cabal-install ghc-template-haskell-devel fi archlinux:latest pacman -S -y --noconfirm cabal-install ghc-static base-devel # Ubuntu LTS +ubuntu:22.04 apt-get update && apt-get install -y cabal-install ubuntu:20.04 apt-get update && apt-get install -y cabal-install +ubuntu:18.04 apt-get update && apt-get install -y cabal-install +ubuntu:16.04 apt-get update && apt-get install -y cabal-install # Stack on Ubuntu LTS ubuntu:20.04 set -e; apt-get update && apt-get install -y curl && curl -sSL https://get.haskellstack.org/ | sh -s - -f && cd /mnt && exec test/stacktest From 8754c21244ada70c070763644044fa166387f708 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 11 Dec 2022 16:37:49 -0800 Subject: [PATCH 597/763] Avoid $ trigger TH --- src/ShellCheck/Formatter/CheckStyle.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Formatter/CheckStyle.hs b/src/ShellCheck/Formatter/CheckStyle.hs index c79ac21..6ad6c9c 100644 --- a/src/ShellCheck/Formatter/CheckStyle.hs +++ b/src/ShellCheck/Formatter/CheckStyle.hs @@ -88,7 +88,7 @@ outputError file error = putStrLn $ concat [ attr s v = concat [ s, "='", escape v, "' " ] escape = concatMap escape' escape' c = if isOk c then [c] else "&#" ++ show (ord c) ++ ";" -isOk x = any ($x) [isAsciiUpper, isAsciiLower, isDigit, (`elem` " ./")] +isOk x = any ($ x) [isAsciiUpper, isAsciiLower, isDigit, (`elem` " ./")] severity "error" = "error" severity "warning" = "warning" From a7c5be93dcbd4c219615e44030073158df4426e5 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 11 Dec 2022 12:29:05 -0800 Subject: [PATCH 598/763] Tighten bounds on packages --- ShellCheck.cabal | 61 ++++++++++++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 28 deletions(-) diff --git a/ShellCheck.cabal b/ShellCheck.cabal index abb32d0..dab588c 100644 --- a/ShellCheck.cabal +++ b/ShellCheck.cabal @@ -45,21 +45,26 @@ library build-depends: semigroups build-depends: - aeson, - array, - base >= 4.8.0.0 && < 5, - bytestring, - containers >= 0.5, - deepseq >= 1.4.0.0, - Diff >= 0.2.0, - directory >= 1.2.3.0, - fgl, - filepath, - mtl >= 2.2.1, - parsec, - QuickCheck >= 2.7.4, - regex-tdfa, - transformers, + -- The lower bounds are based on GHC 7.10.3 + -- The upper bounds are based on GHC 9.4.3 + aeson >= 1.4.0 && < 2.2, + array >= 0.5.1 && < 0.6, + base >= 4.8.0.0 && < 5, + bytestring >= 0.10.6 && < 0.12, + containers >= 0.5.6 && < 0.7, + deepseq >= 1.4.1 && < 1.5, + Diff >= 0.4.0 && < 0.5, + fgl >= 5.7.0 && < 5.9, + filepath >= 1.4.0 && < 1.5, + mtl >= 2.2.2 && < 2.3, + parsec >= 3.1.14 && < 3.2, + QuickCheck >= 2.14.2 && < 2.15, + regex-tdfa >= 1.2.0 && < 1.4, + transformers >= 0.4.2 && < 0.6, + + -- getXdgDirectory from 1.2.3.0 + directory >= 1.2.3 && < 1.4, + -- When cabal supports it, move this to setup-depends: process exposed-modules: @@ -101,17 +106,17 @@ executable shellcheck build-depends: aeson, array, - base >= 4 && < 5, + base, bytestring, containers, - deepseq >= 1.4.0.0, - Diff >= 0.2.0, - directory >= 1.2.3.0, + deepseq, + Diff, + directory, fgl, - mtl >= 2.2.1, + mtl, filepath, - parsec >= 3.0, - QuickCheck >= 2.7.4, + parsec, + QuickCheck, regex-tdfa, transformers, ShellCheck @@ -123,17 +128,17 @@ test-suite test-shellcheck build-depends: aeson, array, - base >= 4 && < 5, + base, bytestring, containers, - deepseq >= 1.4.0.0, - Diff >= 0.2.0, - directory >= 1.2.3.0, + deepseq, + Diff, + directory, fgl, filepath, - mtl >= 2.2.1, + mtl, parsec, - QuickCheck >= 2.7.4, + QuickCheck, regex-tdfa, transformers, ShellCheck From 7cfcf6db8a3d5a15cfd361293bbbe2d38d473c5e Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 11 Dec 2022 19:22:42 -0800 Subject: [PATCH 599/763] Fix stack build --- test/distrotest | 2 +- test/stacktest | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/distrotest b/test/distrotest index e1711ea..53a40a7 100755 --- a/test/distrotest +++ b/test/distrotest @@ -73,7 +73,7 @@ ubuntu:18.04 apt-get update && apt-get install -y cabal-install ubuntu:16.04 apt-get update && apt-get install -y cabal-install # Stack on Ubuntu LTS -ubuntu:20.04 set -e; apt-get update && apt-get install -y curl && curl -sSL https://get.haskellstack.org/ | sh -s - -f && cd /mnt && exec test/stacktest +ubuntu:22.04 set -e; apt-get update && apt-get install -y curl && curl -sSL https://get.haskellstack.org/ | sh -s - -f && cd /mnt && exec test/stacktest EOF exit "$final" diff --git a/test/stacktest b/test/stacktest index ae04f1b..9eb8d1e 100755 --- a/test/stacktest +++ b/test/stacktest @@ -3,7 +3,7 @@ # various resolvers. It's run via distrotest. resolvers=( - nightly-"$(date -d "3 days ago" +"%Y-%m-%d")" +# nightly-"$(date -d "3 days ago" +"%Y-%m-%d")" ) die() { echo "$*" >&2; exit 1; } From ae199edb680dd416790b7890fa52e64a11f2b4af Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 11 Dec 2022 20:50:33 -0800 Subject: [PATCH 600/763] Let distrotest fail fast when there remaining executables --- test/distrotest | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/distrotest b/test/distrotest index 53a40a7..4ad66f8 100755 --- a/test/distrotest +++ b/test/distrotest @@ -25,6 +25,13 @@ exit 0 echo "Deleting 'dist' and 'dist-newstyle'..." rm -rf dist dist-newstyle +execs=$(find . -name shellcheck) + +if [ -n "$execs" ] +then + die "Found unexpected executables. Remove and try again: $execs" +fi + log=$(mktemp) || die "Can't create temp file" date >> "$log" || die "Can't write to log" From 8c5fdc3522236767e9ce840b57da2a13b32eb4ed Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 12 Dec 2022 21:49:01 -0800 Subject: [PATCH 601/763] Update copyright years --- shellcheck.1.md | 2 +- src/ShellCheck/Analytics.hs | 2 +- src/ShellCheck/Analyzer.hs | 2 +- src/ShellCheck/AnalyzerLib.hs | 2 +- src/ShellCheck/Checker.hs | 2 +- src/ShellCheck/Checks/Commands.hs | 2 +- src/ShellCheck/Parser.hs | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/shellcheck.1.md b/shellcheck.1.md index c345a2b..9675e79 100644 --- a/shellcheck.1.md +++ b/shellcheck.1.md @@ -378,7 +378,7 @@ long list of wonderful contributors. # COPYRIGHT -Copyright 2012-2021, Vidar Holen and contributors. +Copyright 2012-2022, Vidar Holen and contributors. Licensed under the GNU General Public License version 3 or later, see https://gnu.org/licenses/gpl.html diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index f50510d..1f6d96d 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1,5 +1,5 @@ {- - Copyright 2012-2021 Vidar Holen + Copyright 2012-2022 Vidar Holen This file is part of ShellCheck. https://www.shellcheck.net diff --git a/src/ShellCheck/Analyzer.hs b/src/ShellCheck/Analyzer.hs index 06b6e53..53717ed 100644 --- a/src/ShellCheck/Analyzer.hs +++ b/src/ShellCheck/Analyzer.hs @@ -1,5 +1,5 @@ {- - Copyright 2012-2019 Vidar Holen + Copyright 2012-2022 Vidar Holen This file is part of ShellCheck. https://www.shellcheck.net diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 47ea91d..ca928fd 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -1,5 +1,5 @@ {- - Copyright 2012-2021 Vidar Holen + Copyright 2012-2022 Vidar Holen This file is part of ShellCheck. https://www.shellcheck.net diff --git a/src/ShellCheck/Checker.hs b/src/ShellCheck/Checker.hs index c8d2c39..b56be68 100644 --- a/src/ShellCheck/Checker.hs +++ b/src/ShellCheck/Checker.hs @@ -1,5 +1,5 @@ {- - Copyright 2012-2020 Vidar Holen + Copyright 2012-2022 Vidar Holen This file is part of ShellCheck. https://www.shellcheck.net diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index e97ecd6..691836f 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -1,5 +1,5 @@ {- - Copyright 2012-2021 Vidar Holen + Copyright 2012-2022 Vidar Holen This file is part of ShellCheck. https://www.shellcheck.net diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index dd0f0f0..7a50967 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -1,5 +1,5 @@ {- - Copyright 2012-2021 Vidar Holen + Copyright 2012-2022 Vidar Holen This file is part of ShellCheck. https://www.shellcheck.net From a526ee08290cc127bc1aa5a05e9b927af87b6ef3 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 12 Dec 2022 19:58:11 -0800 Subject: [PATCH 602/763] Stable version 0.9.0 This release is dedicated to Mindustry: the most fun you can have with open source (outside of shell scripting of course). --- CHANGELOG.md | 2 +- ShellCheck.cabal | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c363eb5..57951c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## Git +## v0.9.0 - 2022-12-12 ### Added - SC2316: Warn about 'local readonly foo' and similar (thanks, patrickxia!) - SC2317: Warn about unreachable commands diff --git a/ShellCheck.cabal b/ShellCheck.cabal index dab588c..1226588 100644 --- a/ShellCheck.cabal +++ b/ShellCheck.cabal @@ -1,5 +1,5 @@ Name: ShellCheck -Version: 0.8.0 +Version: 0.9.0 Synopsis: Shell script analysis tool License: GPL-3 License-file: LICENSE From 5a3eb89e385d3667ecda33cfe769571ffc9dd7a3 Mon Sep 17 00:00:00 2001 From: Samuel Lijin Date: Fri, 3 Feb 2023 09:17:47 -0800 Subject: [PATCH 603/763] Document Trunk Check integration Trunk Check is a universal linter which integrates with a wide variety of linters and formatters, `shellcheck` included. We're big fans of `shellcheck` and figured that you might find our tool to be interesting enough to include it in the integrations list. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 6f3e4a9..8cf3584 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,7 @@ Services and platforms that have ShellCheck pre-installed and ready to use: * [Code Factor](https://www.codefactor.io/) * [CircleCI](https://circleci.com) via the [ShellCheck Orb](https://circleci.com/orbs/registry/orb/circleci/shellcheck) * [Github](https://github.com/features/actions) (only Linux) +* [Trunk Check](https://trunk.io/products/check) (universal linter; [allows you to explicitly version your shellcheck install](https://github.com/trunk-io/plugins/blob/bcbb361dcdbe4619af51ea7db474d7fb87540d20/.trunk/trunk.yaml#L32)) via the [shellcheck plugin](https://github.com/trunk-io/plugins/blob/main/linters/shellcheck/plugin.yaml) Most other services, including [GitLab](https://about.gitlab.com/), let you install ShellCheck yourself, either through the system's package manager (see [Installing](#installing)), From 78dea1d4f93987c7e52df121f8062da3387d0f2e Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 4 Feb 2023 10:27:59 -0800 Subject: [PATCH 604/763] Update changelog from release --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 57951c8..fc1c0ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +## Git +### Added + +### Fixed + +### Changed + + ## v0.9.0 - 2022-12-12 ### Added - SC2316: Warn about 'local readonly foo' and similar (thanks, patrickxia!) From 2842ce97b88a9cd9551f5f42f67a97e17a523b53 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 4 Feb 2023 11:38:20 -0800 Subject: [PATCH 605/763] Remove fgl-5.8.1.0 as a dependency ShellCheck is temporarily broken by https://github.com/haskell/fgl/commit/c8f56c18242b0c3e916892c5db56a609ec396637 --- ShellCheck.cabal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ShellCheck.cabal b/ShellCheck.cabal index 1226588..8afebe1 100644 --- a/ShellCheck.cabal +++ b/ShellCheck.cabal @@ -54,7 +54,7 @@ library containers >= 0.5.6 && < 0.7, deepseq >= 1.4.1 && < 1.5, Diff >= 0.4.0 && < 0.5, - fgl >= 5.7.0 && < 5.9, + fgl >= 5.7.0 && < 5.8.1.0, filepath >= 1.4.0 && < 1.5, mtl >= 2.2.2 && < 2.3, parsec >= 3.1.14 && < 3.2, From c05380d518056189412e12128a8906b8ca6f6717 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 4 Feb 2023 13:19:27 -0800 Subject: [PATCH 606/763] Count CFEExit as control flow for the purposes of finding dominators --- src/ShellCheck/CFG.hs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/CFG.hs b/src/ShellCheck/CFG.hs index e0c6267..f882adc 100644 --- a/src/ShellCheck/CFG.hs +++ b/src/ShellCheck/CFG.hs @@ -192,7 +192,7 @@ buildGraph params root = base idToRange = M.fromList mapping - isRealEdge (from, to, edge) = case edge of CFEFlow -> True; _ -> False + isRealEdge (from, to, edge) = case edge of CFEFlow -> True; CFEExit -> True; _ -> False onlyRealEdges = filter isRealEdge edges (_, mainExit) = fromJust $ M.lookup (getId root) idToRange @@ -1301,7 +1301,10 @@ findPostDominators mainexit graph = asArray reversed = grev withExitEdges postDoms = dom reversed mainexit (_, maxNode) = nodeRange graph - asArray = array (0, maxNode) postDoms + -- Holes in the array cause "Exception: (Array.!): undefined array element" while + -- inspecting/debugging, so fill the array first and then update. + initializedArray = listArray (0, maxNode) $ repeat [] + asArray = initializedArray // postDoms return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) From b1ca3929e387446f3e3db023d716cf3787370437 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 4 Feb 2023 19:55:25 -0800 Subject: [PATCH 607/763] Upgrade cross-compilers to 9.2.5 to handle hashable-1.4.2.0 --- build/darwin.x86_64/Dockerfile | 11 +++++++---- build/linux.aarch64/Dockerfile | 14 ++++++++++---- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/build/darwin.x86_64/Dockerfile b/build/darwin.x86_64/Dockerfile index 9e33a82..a53245f 100644 --- a/build/darwin.x86_64/Dockerfile +++ b/build/darwin.x86_64/Dockerfile @@ -6,15 +6,18 @@ ENV TARGETNAME darwin.x86_64 # Build dependencies USER root ENV DEBIAN_FRONTEND noninteractive -RUN apt-get update && apt-get install -y ghc automake autoconf llvm curl +RUN sed -e 's/focal/kinetic/g' -i /etc/apt/sources.list +RUN apt-get update +RUN apt-get dist-upgrade -y +RUN apt-get install -y ghc automake autoconf llvm curl alex happy # Build GHC WORKDIR /ghc -RUN curl -L "https://downloads.haskell.org/~ghc/8.10.4/ghc-8.10.4-src.tar.xz" | tar xJ --strip-components=1 -RUN ./boot && ./configure --host x86_64-linux-gnu --build x86_64-linux-gnu --target "$TARGET" +RUN curl -L "https://downloads.haskell.org/~ghc/9.2.5/ghc-9.2.5-src.tar.xz" | tar xJ --strip-components=1 +RUN ./configure --host x86_64-linux-gnu --build x86_64-linux-gnu --target "$TARGET" RUN cp mk/flavours/quick-cross.mk mk/build.mk && make -j "$(nproc)" RUN make install -RUN curl -L "https://downloads.haskell.org/~cabal/cabal-install-3.2.0.0/cabal-install-3.2.0.0-x86_64-unknown-linux.tar.xz" | tar xJv -C /usr/local/bin +RUN curl -L "https://downloads.haskell.org/~cabal/cabal-install-3.9.0.0/cabal-install-3.9-x86_64-linux-alpine.tar.xz" | tar xJv -C /usr/local/bin # Due to an apparent cabal bug, we specify our options directly to cabal # It won't reuse caches if ghc-options are specified in ~/.cabal/config diff --git a/build/linux.aarch64/Dockerfile b/build/linux.aarch64/Dockerfile index 60537b3..d5320e9 100644 --- a/build/linux.aarch64/Dockerfile +++ b/build/linux.aarch64/Dockerfile @@ -6,19 +6,25 @@ ENV TARGETNAME linux.aarch64 # Build dependencies USER root ENV DEBIAN_FRONTEND noninteractive -RUN apt-get update && apt-get install -y ghc automake autoconf build-essential llvm curl qemu-user-static gcc-$TARGET + +# These deps are from 20.04, because GHC's compiler/llvm support moves slowly +RUN apt-get update && apt-get install -y llvm gcc-$TARGET + +# The rest are from 22.10 +RUN sed -e 's/focal/kinetic/g' -i /etc/apt/sources.list +RUN apt-get update && apt-get install -y ghc alex happy automake autoconf build-essential curl qemu-user-static # Build GHC WORKDIR /ghc -RUN curl -L "https://downloads.haskell.org/~ghc/8.10.4/ghc-8.10.4-src.tar.xz" | tar xJ --strip-components=1 +RUN curl -L "https://downloads.haskell.org/~ghc/9.2.5/ghc-9.2.5-src.tar.xz" | tar xJ --strip-components=1 RUN ./boot && ./configure --host x86_64-linux-gnu --build x86_64-linux-gnu --target "$TARGET" RUN cp mk/flavours/quick-cross.mk mk/build.mk && make -j "$(nproc)" RUN make install -RUN curl -L "https://downloads.haskell.org/~cabal/cabal-install-3.2.0.0/cabal-install-3.2.0.0-x86_64-unknown-linux.tar.xz" | tar xJv -C /usr/local/bin +RUN curl -L "https://downloads.haskell.org/~cabal/cabal-install-3.9.0.0/cabal-install-3.9-x86_64-linux-alpine.tar.xz" | tar xJv -C /usr/local/bin # Due to an apparent cabal bug, we specify our options directly to cabal # It won't reuse caches if ghc-options are specified in ~/.cabal/config -ENV CABALOPTS "--ghc-options;-split-sections -optc-Os -optc-Wl,--gc-sections;--with-ghc=$TARGET-ghc;--with-hc-pkg=$TARGET-ghc-pkg" +ENV CABALOPTS "--ghc-options;-split-sections -optc-Os -optc-Wl,--gc-sections -optc-fPIC;--with-ghc=$TARGET-ghc;--with-hc-pkg=$TARGET-ghc-pkg" # Prebuild the dependencies RUN cabal update && IFS=';' && cabal install --dependencies-only $CABALOPTS ShellCheck From e6e8ab0415f720afee5f426b6ca09757a4ed12ce Mon Sep 17 00:00:00 2001 From: Felipe Santos Date: Sun, 5 Feb 2023 11:13:07 -0300 Subject: [PATCH 608/763] Mention VS Code ShellCheck binaries distribution --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 6f3e4a9..a1c59a7 100644 --- a/README.md +++ b/README.md @@ -232,6 +232,8 @@ Alternatively, you can download pre-compiled binaries for the latest release her or see the [GitHub Releases](https://github.com/koalaman/shellcheck/releases) for other releases (including the [latest](https://github.com/koalaman/shellcheck/releases/tag/latest) meta-release for daily git builds). +You can also find pre-compiled binaries repackaged in `.tar.gz` format in the [VS Code ShellCheck Binaries](https://github.com/vscode-shellcheck/shellcheck-binaries/releases) repository. It also includes a pre-compiled binary for **Apple M1** processors. + Distro packages already come with a `man` page. If you are building from source, it can be installed with: ```console From 08b437974e871d82213545b9a079bcb794f7a58a Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 23 Apr 2023 16:47:49 -0700 Subject: [PATCH 609/763] Rewrite vscode-shellcheck blurb --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a1c59a7..3a839aa 100644 --- a/README.md +++ b/README.md @@ -232,7 +232,8 @@ Alternatively, you can download pre-compiled binaries for the latest release her or see the [GitHub Releases](https://github.com/koalaman/shellcheck/releases) for other releases (including the [latest](https://github.com/koalaman/shellcheck/releases/tag/latest) meta-release for daily git builds). -You can also find pre-compiled binaries repackaged in `.tar.gz` format in the [VS Code ShellCheck Binaries](https://github.com/vscode-shellcheck/shellcheck-binaries/releases) repository. It also includes a pre-compiled binary for **Apple M1** processors. +There are currently no official binaries for Apple Silicon, but third party builds are available via +[ShellCheck for Visual Studio Code](https://github.com/vscode-shellcheck/shellcheck-binaries/releases). Distro packages already come with a `man` page. If you are building from source, it can be installed with: From 1164aa4efc225129302e2a2450e907ad842d41e4 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 23 Apr 2023 19:35:54 -0700 Subject: [PATCH 610/763] Installing custom docker should no longer be necessary for buildx --- .multi_arch_docker | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/.multi_arch_docker b/.multi_arch_docker index a9f7401..1c5d32b 100755 --- a/.multi_arch_docker +++ b/.multi_arch_docker @@ -3,28 +3,10 @@ # binaries previously built and deployed to GitHub. function multi_arch_docker::install_docker_buildx() { - # Install up-to-date version of docker, with buildx support. - local -r docker_apt_repo='https://download.docker.com/linux/ubuntu' - curl -fsSL "${docker_apt_repo}/gpg" | sudo apt-key add - - local -r os="$(lsb_release -cs)" - sudo add-apt-repository "deb [arch=amd64] $docker_apt_repo $os stable" - sudo apt-get update - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce - - # Enable docker daemon experimental support (for 'pull --platform'). - local -r config='/etc/docker/daemon.json' - if [[ -e "$config" ]]; then - sudo sed -i -e 's/{/{ "experimental": true, /' "$config" - else - echo '{ "experimental": true }' | sudo tee "$config" - fi - sudo systemctl restart docker - # Install QEMU multi-architecture support for docker buildx. docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # Instantiate docker buildx builder with multi-architecture support. - export DOCKER_CLI_EXPERIMENTAL=enabled docker buildx create --name mybuilder docker buildx use mybuilder # Start up buildx and verify that all is OK. From 5fec3f9b34bc0ba3d2d02d4a7613cb3a62ed4c15 Mon Sep 17 00:00:00 2001 From: James Morris Date: Mon, 24 Apr 2023 22:08:22 -0400 Subject: [PATCH 611/763] Add fish to the badShells list --- src/ShellCheck/Parser.hs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 7a50967..9d7df00 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -3360,6 +3360,7 @@ readScriptFile sourced = do "awk", "csh", "expect", + "fish", "perl", "python", "ruby", From 46b678fca8f8aac035d04e676f77f1a92f6742f4 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 30 Apr 2023 14:37:37 -0700 Subject: [PATCH 612/763] Minor fixes to POSIX read without variable check --- src/ShellCheck/Checks/ShellSupport.hs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index eda6882..cf8acc9 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -188,6 +188,7 @@ prop_checkBashisms100 = verify checkBashisms "read -r" prop_checkBashisms101 = verify checkBashisms "read" prop_checkBashisms102 = verifyNot checkBashisms "read -r foo" prop_checkBashisms103 = verifyNot checkBashisms "read foo" +prop_checkBashisms104 = verifyNot checkBashisms "read ''" checkBashisms = ForShell [Sh, Dash] $ \t -> do params <- ask kludge params t @@ -288,13 +289,6 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do argString = concat $ oversimplify arg flagRegex = mkRegex "^-[eEsn]+$" - bashism t@(T_SimpleCommand _ _ (cmd:args)) - | t `isCommand` "read" && length (onlyNames args) == 0 = - warnMsg (getId cmd) 3061 "read without a variable is" - where - notFlag arg = head (concat $ oversimplify arg) /= '-' - onlyNames = filter (notFlag) - bashism t@(T_SimpleCommand _ _ (cmd:arg:_)) | getLiteralString cmd == Just "exec" && "-" `isPrefixOf` concat (oversimplify arg) = warnMsg (getId arg) 3038 "exec flags are" @@ -390,6 +384,9 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do let literal = onlyLiteralString format guard $ "%q" `isInfixOf` literal return $ warnMsg (getId format) 3050 "printf %q is" + + when (name == "read" && all isFlag rest) $ + warnMsg (getId cmd) 3061 "read without a variable is" where unsupportedCommands = [ "let", "caller", "builtin", "complete", "compgen", "declare", "dirs", "disown", From b3932dfa10804434fb8c15dc32e428c5a1c3bfa4 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Mon, 1 May 2023 00:02:53 -0400 Subject: [PATCH 613/763] Fix #2734: adjust bounds to compile on 9.6 The whole test suite passes for me, including prop_checkOverwrittenExitCode8, and I get the same set of findings with this build and shellcheck.net on tools/testing/selftests/net/icmp_redirect.sh. --- ShellCheck.cabal | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ShellCheck.cabal b/ShellCheck.cabal index 8afebe1..f09521f 100644 --- a/ShellCheck.cabal +++ b/ShellCheck.cabal @@ -46,7 +46,7 @@ library semigroups build-depends: -- The lower bounds are based on GHC 7.10.3 - -- The upper bounds are based on GHC 9.4.3 + -- The upper bounds are based on GHC 9.6.1 aeson >= 1.4.0 && < 2.2, array >= 0.5.1 && < 0.6, base >= 4.8.0.0 && < 5, @@ -54,13 +54,13 @@ library containers >= 0.5.6 && < 0.7, deepseq >= 1.4.1 && < 1.5, Diff >= 0.4.0 && < 0.5, - fgl >= 5.7.0 && < 5.8.1.0, + fgl (>= 5.7.0 && < 5.8.1.0) || (>= 5.8.1.1 && < 5.9), filepath >= 1.4.0 && < 1.5, - mtl >= 2.2.2 && < 2.3, + mtl >= 2.2.2 && < 2.4, parsec >= 3.1.14 && < 3.2, QuickCheck >= 2.14.2 && < 2.15, regex-tdfa >= 1.2.0 && < 1.4, - transformers >= 0.4.2 && < 0.6, + transformers >= 0.4.2 && < 0.7, -- getXdgDirectory from 1.2.3.0 directory >= 1.2.3 && < 1.4, From f03c437e2fe0d669d1e64dcedd305d5bd8ca0608 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Wed, 24 May 2023 16:38:53 -0400 Subject: [PATCH 614/763] Get rid of a dangerous partial function from checkSpacefulnessCfg' --- src/ShellCheck/Analytics.hs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 1f6d96d..ecc170d 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2134,7 +2134,8 @@ checkSpacefulnessCfg' dirtyPass params token@(T_DollarBraced id _ list) = addDoubleQuotesAround params token where - name = getBracedReference $ concat $ oversimplify list + bracedString = concat $ oversimplify list + name = getBracedReference bracedString parents = parentMap params needsQuoting = not (isArrayExpansion token) -- There's another warning for this @@ -2153,14 +2154,10 @@ checkSpacefulnessCfg' dirtyPass params token@(T_DollarBraced id _ list) = || CF.spaceStatus (CF.variableValue state) == CF.SpaceStatusClean isDefaultAssignment parents token = - let modifier = getBracedModifier $ bracedString token in + let modifier = getBracedModifier bracedString in any (`isPrefixOf` modifier) ["=", ":="] && isParamTo parents ":" token - -- Given a T_DollarBraced, return a simplified version of the string contents. - bracedString (T_DollarBraced _ _ l) = concat $ oversimplify l - bracedString _ = error $ pleaseReport "bracedString on non-variable" - checkSpacefulnessCfg' _ _ _ = return () From b625cc1accb3249322ca757b6709840f7582b072 Mon Sep 17 00:00:00 2001 From: Nicolas Theodarus Date: Sun, 28 May 2023 12:33:16 +0200 Subject: [PATCH 615/763] add dependabot.yml --- .github/dependabot.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..81bae9a --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,7 @@ +version: 2 + +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" From 01aee1a859913a02b529d1f25469693166d3fe7c Mon Sep 17 00:00:00 2001 From: Danny Faught Date: Fri, 28 Jul 2023 14:19:54 -0400 Subject: [PATCH 616/763] improve short description * The short description used to say that until commit aac7d76047a5b28d064b17a5d0fac022054d05a0 from 2014. It appears that it was changed by mistake in that commit to something less readable. * With the message "use -print0/-0" we were confused and introduced a bug in our code because we didn't understand what to do with the "-0". * SC2011 (source https://github.com/koalaman/shellcheck/blob/c9e27c24700cdc5b84cfca1f7a90fe07f542867c/src/ShellCheck/Analytics.hs#L591) uses that exact warning message, we copied it from there. Signed-off-by: Bruce Ricard --- src/ShellCheck/Analytics.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index ecc170d..3bb1ed0 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -562,7 +562,7 @@ checkPipePitfalls _ (T_Pipeline id _ commands) = do hasParameter "print0", hasParameter "printf" ]) $ warn (getId find) 2038 - "Use -print0/-0 or -exec + to allow for non-alphanumeric filenames." + "Use 'find .. -print0 | xargs -0 ..' or 'find .. -exec .. +' to allow non-alphanumeric filenames." for ["ps", "grep"] $ \(ps:grep:_) -> From 372c0b667e7b6f36a5f1a42a9802eb0246ee3e95 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 30 Jul 2023 13:47:00 -0700 Subject: [PATCH 617/763] SC2324: Warn when x+=1 appends. --- CHANGELOG.md | 1 + src/ShellCheck/ASTLib.hs | 9 +++++++++ src/ShellCheck/Analytics.hs | 38 +++++++++++++++++++++++++++++++++++ src/ShellCheck/CFGAnalysis.hs | 16 +++++++++++++++ 4 files changed, 64 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fc1c0ce..8f4426e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ## Git ### Added +- SC2324: Warn when x+=1 appends instead of increments. ### Fixed diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 56903ee..64fa762 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -886,6 +886,15 @@ isUnmodifiedParameterExpansion t = in getBracedReference str == str _ -> False +-- Return the referenced variable if (and only if) it's an unmodified parameter expansion. +getUnmodifiedParameterExpansion t = + case t of + T_DollarBraced _ _ list -> do + let str = concat $ oversimplify list + guard $ getBracedReference str == str + return str + _ -> Nothing + --- A list of the element and all its parents up to the root node. getPath tree t = t : case Map.lookup (getId t) tree of diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index ecc170d..dbad0f5 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -201,6 +201,7 @@ nodeChecks = [ ,checkOverwrittenExitCode ,checkUnnecessaryArithmeticExpansionIndex ,checkUnnecessaryParens + ,checkPlusEqualsNumber ] optionalChecks = map fst optionalTreeChecks @@ -5007,5 +5008,42 @@ checkUnnecessaryParens params t = ] +prop_checkPlusEqualsNumber1 = verify checkPlusEqualsNumber "x+=1" +prop_checkPlusEqualsNumber2 = verify checkPlusEqualsNumber "x+=42" +prop_checkPlusEqualsNumber3 = verifyNot checkPlusEqualsNumber "(( x += 1 ))" +prop_checkPlusEqualsNumber4 = verifyNot checkPlusEqualsNumber "declare -i x=0; x+=1" +prop_checkPlusEqualsNumber5 = verifyNot checkPlusEqualsNumber "x+='1'" +prop_checkPlusEqualsNumber6 = verifyNot checkPlusEqualsNumber "n=foo; x+=n" +prop_checkPlusEqualsNumber7 = verify checkPlusEqualsNumber "n=4; x+=n" +prop_checkPlusEqualsNumber8 = verify checkPlusEqualsNumber "n=4; x+=$n" +prop_checkPlusEqualsNumber9 = verifyNot checkPlusEqualsNumber "declare -ia var; var[x]+=1" +checkPlusEqualsNumber params t = + case t of + T_Assignment id Append var _ word -> sequence_ $ do + state <- CF.getIncomingState (cfgAnalysis params) id + guard $ isNumber state word + guard . not $ fromMaybe False $ CF.variableMayBeDeclaredInteger state var + return $ warn id 2324 "var+=1 will append, not increment. Use (( var += 1 )), declare -i var, or quote number to silence." + _ -> return () + + where + isNumber state word = + let + unquotedLiteral = getUnquotedLiteral word + isEmpty = unquotedLiteral == Just "" + isUnquotedNumber = not isEmpty && fromMaybe False (all isDigit <$> unquotedLiteral) + isNumericalVariableName = fromMaybe False $ do + str <- unquotedLiteral + CF.variableMayBeAssignedInteger state str + isNumericalVariableExpansion = + case word of + T_NormalWord _ [part] -> fromMaybe False $ do + str <- getUnmodifiedParameterExpansion part + CF.variableMayBeAssignedInteger state str + _ -> False + in + isUnquotedNumber || isNumericalVariableName || isNumericalVariableExpansion + + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) diff --git a/src/ShellCheck/CFGAnalysis.hs b/src/ShellCheck/CFGAnalysis.hs index 4e36cf5..3b4f957 100644 --- a/src/ShellCheck/CFGAnalysis.hs +++ b/src/ShellCheck/CFGAnalysis.hs @@ -59,6 +59,8 @@ module ShellCheck.CFGAnalysis ( ,getIncomingState ,getOutgoingState ,doesPostDominate + ,variableMayBeDeclaredInteger + ,variableMayBeAssignedInteger ,ShellCheck.CFGAnalysis.runTests -- STRIP ) where @@ -153,6 +155,20 @@ doesPostDominate analysis target base = fromMaybe False $ do (targetStart, _) <- M.lookup target $ tokenToRange analysis return $ targetStart `elem` (postDominators analysis ! baseEnd) +-- See if any execution path results in the variable containing a state +variableMayHaveState :: ProgramState -> String -> CFVariableProp -> Maybe Bool +variableMayHaveState state var property = do + value <- M.lookup var $ variablesInScope state + return $ any (S.member property) $ variableProperties value + +-- See if any execution path declares the variable an integer (declare -i). +variableMayBeDeclaredInteger state var = variableMayHaveState state var CFVPInteger + +-- See if any execution path suggests the variable may contain an integer value +variableMayBeAssignedInteger state var = do + value <- M.lookup var $ variablesInScope state + return $ (numericalStatus $ variableValue value) >= NumericalStatusMaybe + getDataForNode analysis node = M.lookup node $ nodeToData analysis -- The current state of data flow at a point in the program, potentially as a diff From 9490b9488627a06e0a4af1c11644b7936b8a2422 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 30 Jul 2023 16:52:40 -0700 Subject: [PATCH 618/763] Save and restore pending here docs when sourcing files (fixes #2803) --- CHANGELOG.md | 3 ++- src/ShellCheck/Checker.hs | 9 +++++++++ src/ShellCheck/Parser.hs | 10 ++++++++-- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f4426e..c6c9513 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,9 @@ ## Git ### Added -- SC2324: Warn when x+=1 appends instead of increments. +- SC2324: Warn when x+=1 appends instead of increments ### Fixed +- source statements with here docs now work correctly ### Changed diff --git a/src/ShellCheck/Checker.hs b/src/ShellCheck/Checker.hs index b56be68..c79f90f 100644 --- a/src/ShellCheck/Checker.hs +++ b/src/ShellCheck/Checker.hs @@ -508,5 +508,14 @@ prop_rcCanSuppressEarlyProblems2 = null result csScript = "!/bin/bash\necho 'hello world'" } +prop_sourceWithHereDocWorks = null result + where + result = checkWithIncludes [("bar", "true\n")] "source bar << eof\nlol\neof" + +prop_hereDocsAreParsedWithoutTrailingLinefeed = 1044 `elem` result + where + result = check "cat << eof" + + return [] runTests = $quickCheckAll diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 9d7df00..341a435 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -2283,8 +2283,13 @@ readSource t@(T_Redirecting _ _ (T_SimpleCommand cmdId _ (cmd:file':rest'))) = d subRead name script = withContext (ContextSource name) $ - inSeparateContext $ - subParse (initialPos name) (readScriptFile True) script + inSeparateContext $ do + oldState <- getState + setState $ oldState { pendingHereDocs = [] } + result <- subParse (initialPos name) (readScriptFile True) script + newState <- getState + setState $ newState { pendingHereDocs = pendingHereDocs oldState } + return result readSource t = return t @@ -3322,6 +3327,7 @@ readScriptFile sourced = do then do commands <- readCompoundListOrEmpty id <- endSpan start + readPendingHereDocs verifyEof let script = T_Annotation annotationId annotations $ T_Script id shebang commands From dd747b2a98c3214978a97b9ee0ec38e635b6e621 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 30 Jul 2023 19:18:27 -0700 Subject: [PATCH 619/763] SC2325/SC2326: Warn about ! ! foo and foo | ! bar (fixes #2810) --- CHANGELOG.md | 2 ++ src/ShellCheck/Checks/ShellSupport.hs | 26 ++++++++++++++++++++++++++ src/ShellCheck/Parser.hs | 22 +++++++++++++++------- 3 files changed, 43 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c6c9513..0338f7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ ## Git ### Added - SC2324: Warn when x+=1 appends instead of increments +- SC2325: Warn about multiple `!`s in dash/sh. +- SC2326: Warn about `foo | ! bar` in bash/dash/sh. ### Fixed - source statements with here docs now work correctly diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index cf8acc9..c7ece1a 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -60,6 +60,8 @@ checks = [ ,checkBraceExpansionVars ,checkMultiDimensionalArrays ,checkPS1Assignments + ,checkMultipleBangs + ,checkBangAfterPipe ] testChecker (ForShell _ t) = @@ -566,5 +568,29 @@ checkPS1Assignments = ForShell [Bash] f escapeRegex = mkRegex "\\\\x1[Bb]|\\\\e|\x1B|\\\\033" +prop_checkMultipleBangs1 = verify checkMultipleBangs "! ! true" +prop_checkMultipleBangs2 = verifyNot checkMultipleBangs "! true" +checkMultipleBangs = ForShell [Dash, Sh] f + where + f token = case token of + T_Banged id (T_Banged _ _) -> + err id 2325 "Multiple ! in front of pipelines are a bash/ksh extension. Use only 0 or 1." + _ -> return () + + +prop_checkBangAfterPipe1 = verify checkBangAfterPipe "true | ! true" +prop_checkBangAfterPipe2 = verifyNot checkBangAfterPipe "true | ( ! true )" +prop_checkBangAfterPipe3 = verifyNot checkBangAfterPipe "! ! true | true" +checkBangAfterPipe = ForShell [Dash, Sh, Bash] f + where + f token = case token of + T_Pipeline _ _ cmds -> mapM_ check cmds + _ -> return () + + check token = case token of + T_Banged id _ -> + err id 2326 "! is not allowed in the middle of pipelines. Use command group as in cmd | { ! cmd; } if necessary." + _ -> return () + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 341a435..ffc58e2 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -2296,14 +2296,18 @@ readSource t = return t prop_readPipeline = isOk readPipeline "! cat /etc/issue | grep -i ubuntu" prop_readPipeline2 = isWarning readPipeline "!cat /etc/issue | grep -i ubuntu" prop_readPipeline3 = isOk readPipeline "for f; do :; done|cat" +prop_readPipeline4 = isOk readPipeline "! ! true" +prop_readPipeline5 = isOk readPipeline "true | ! true" readPipeline = do unexpecting "keyword/token" readKeyword - do - (T_Bang id) <- g_Bang - pipe <- readPipeSequence - return $ T_Banged id pipe - <|> - readPipeSequence + readBanged readPipeSequence + +readBanged parser = do + pos <- getPosition + (T_Bang id) <- g_Bang + next <- readBanged parser + return $ T_Banged id next + <|> parser prop_readAndOr = isOk readAndOr "grep -i lol foo || exit 1" prop_readAndOr1 = isOk readAndOr "# shellcheck disable=1\nfoo" @@ -2359,7 +2363,7 @@ readTerm = do readPipeSequence = do start <- startSpan - (cmds, pipes) <- sepBy1WithSeparators readCommand + (cmds, pipes) <- sepBy1WithSeparators (readBanged readCommand) (readPipe `thenSkip` (spacing >> readLineBreak)) id <- endSpan start spacing @@ -2389,6 +2393,10 @@ readCommand = choice [ ] readCmdName = do + -- If the command name is `!` then + optional . lookAhead . try $ do + char '!' + whitespace -- Ignore alias suppression optional . try $ do char '\\' From 90d3172dfec30a7569f95b32479ae97af73b8b2e Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 13 Aug 2023 16:32:53 -0700 Subject: [PATCH 620/763] Add a newSystemInterface to go with the rest of the new* constructors --- shellcheck.hs | 2 +- src/ShellCheck/Interface.hs | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/shellcheck.hs b/shellcheck.hs index 4e8a155..6be9bb1 100644 --- a/shellcheck.hs +++ b/shellcheck.hs @@ -396,7 +396,7 @@ ioInterface options files = do inputs <- mapM normalize files cache <- newIORef emptyCache configCache <- newIORef ("", Nothing) - return SystemInterface { + return (newSystemInterface :: SystemInterface IO) { siReadFile = get cache inputs, siFindSource = findSourceFile inputs (sourcePaths options), siGetConfig = getConfig configCache diff --git a/src/ShellCheck/Interface.hs b/src/ShellCheck/Interface.hs index 7528559..077212f 100644 --- a/src/ShellCheck/Interface.hs +++ b/src/ShellCheck/Interface.hs @@ -39,11 +39,12 @@ module ShellCheck.Interface , ColorOption(ColorAuto, ColorAlways, ColorNever) , TokenComment(tcId, tcComment, tcFix) , emptyCheckResult - , newParseResult - , newAnalysisSpec , newAnalysisResult + , newAnalysisSpec , newFormatterOptions + , newParseResult , newPosition + , newSystemInterface , newTokenComment , mockedSystemInterface , mockRcFile @@ -135,6 +136,14 @@ newParseSpec = ParseSpec { psShellTypeOverride = Nothing } +newSystemInterface :: Monad m => SystemInterface m +newSystemInterface = + SystemInterface { + siReadFile = \_ _ -> return $ Left "Not implemented", + siFindSource = \_ _ _ name -> return name, + siGetConfig = \_ -> return Nothing + } + -- Parser input and output data ParseSpec = ParseSpec { psFilename :: String, @@ -311,7 +320,7 @@ data ColorOption = -- For testing mockedSystemInterface :: [(String, String)] -> SystemInterface Identity -mockedSystemInterface files = SystemInterface { +mockedSystemInterface files = (newSystemInterface :: SystemInterface Identity) { siReadFile = rf, siFindSource = fs, siGetConfig = const $ return Nothing From 410ec54617c86586cd6b5662ecc035898bd8aefa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 5 Sep 2023 08:21:55 +0000 Subject: [PATCH 621/763] Bump actions/checkout from 3 to 4 Bumps [actions/checkout](https://github.com/actions/checkout) from 3 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a435cf4..2ca94f2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,7 +15,7 @@ jobs: sudo apt-get install cabal-install - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 @@ -51,7 +51,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Download artifacts uses: actions/download-artifact@v3 @@ -74,7 +74,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Download artifacts uses: actions/download-artifact@v3 @@ -104,7 +104,7 @@ jobs: environment: Deploy steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Download artifacts uses: actions/download-artifact@v3 From c89ec2fd492d37722f2c93aaae500cc91a84e1c4 Mon Sep 17 00:00:00 2001 From: Max Ulidtko Date: Sun, 1 Oct 2023 19:57:19 +0200 Subject: [PATCH 622/763] Fix: do []-related bashism checks on test(1) calls too --- src/ShellCheck/Checks/ShellSupport.hs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index c7ece1a..1f012d2 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -19,6 +19,7 @@ -} {-# LANGUAGE TemplateHaskell #-} {-# LANGUAGE FlexibleContexts #-} +{-# LANGUAGE ViewPatterns #-} module ShellCheck.Checks.ShellSupport (checker , ShellCheck.Checks.ShellSupport.runTests) where import ShellCheck.AST @@ -91,6 +92,9 @@ prop_checkBashisms3 = verify checkBashisms "echo $((i++))" prop_checkBashisms4 = verify checkBashisms "rm !(*.hs)" prop_checkBashisms5 = verify checkBashisms "source file" prop_checkBashisms6 = verify checkBashisms "[ \"$a\" == 42 ]" +prop_checkBashisms6b = verify checkBashisms "test \"$a\" == 42" +prop_checkBashisms6c = verify checkBashisms "[ foo =~ bar ]" +prop_checkBashisms6d = verify checkBashisms "test foo =~ bar" prop_checkBashisms7 = verify checkBashisms "echo ${var[1]}" prop_checkBashisms8 = verify checkBashisms "echo ${!var[@]}" prop_checkBashisms9 = verify checkBashisms "echo ${!var*}" @@ -106,6 +110,7 @@ prop_checkBashisms18 = verify checkBashisms "foo &> /dev/null" prop_checkBashisms19 = verify checkBashisms "foo > file*.txt" prop_checkBashisms20 = verify checkBashisms "read -ra foo" prop_checkBashisms21 = verify checkBashisms "[ -a foo ]" +prop_checkBashisms21b = verify checkBashisms "test -a foo" prop_checkBashisms22 = verifyNot checkBashisms "[ foo -a bar ]" prop_checkBashisms23 = verify checkBashisms "trap mything ERR INT" prop_checkBashisms24 = verifyNot checkBashisms "trap mything INT TERM" @@ -203,6 +208,7 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do if isDash then err id code $ "In dash, " ++ s ++ " not supported." else warn id code $ "In POSIX sh, " ++ s ++ " undefined." + asStr = getLiteralString bashism (T_ProcSub id _ _) = warnMsg id 3001 "process substitution is" bashism (T_Extglob id _ _) = warnMsg id 3002 "extglob is" @@ -218,17 +224,31 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do bashism (TC_Binary id SingleBracket op _ _) | op `elem` [ "<", ">", "\\<", "\\>", "<=", ">=", "\\<=", "\\>="] = unless isDash $ warnMsg id 3012 $ "lexicographical " ++ op ++ " is" + bashism (T_SimpleCommand id _ [asStr -> Just "test", lhs, asStr -> Just op, rhs]) + | op `elem` [ "<", ">", "\\<", "\\>", "<=", ">=", "\\<=", "\\>="] = + unless isDash $ warnMsg id 3012 $ "lexicographical " ++ op ++ " is" bashism (TC_Binary id SingleBracket op _ _) | op `elem` [ "-ot", "-nt", "-ef" ] = unless isDash $ warnMsg id 3013 $ op ++ " is" + bashism (T_SimpleCommand id _ [asStr -> Just "test", lhs, asStr -> Just op, rhs]) + | op `elem` [ "-ot", "-nt", "-ef" ] = + unless isDash $ warnMsg id 3013 $ op ++ " is" bashism (TC_Binary id SingleBracket "==" _ _) = warnMsg id 3014 "== in place of = is" + bashism (T_SimpleCommand id _ [asStr -> Just "test", lhs, asStr -> Just "==", rhs]) = + warnMsg id 3014 "== in place of = is" bashism (TC_Binary id SingleBracket "=~" _ _) = warnMsg id 3015 "=~ regex matching is" + bashism (T_SimpleCommand id _ [asStr -> Just "test", lhs, asStr -> Just "=~", rhs]) = + warnMsg id 3015 "=~ regex matching is" bashism (TC_Unary id SingleBracket "-v" _) = warnMsg id 3016 "unary -v (in place of [ -n \"${var+x}\" ]) is" + bashism (T_SimpleCommand id _ [asStr -> Just "test", asStr -> Just "-v", _]) = + warnMsg id 3016 "unary -v (in place of [ -n \"${var+x}\" ]) is" bashism (TC_Unary id _ "-a" _) = warnMsg id 3017 "unary -a in place of -e is" + bashism (T_SimpleCommand id _ [asStr -> Just "test", asStr -> Just "-a", _]) = + warnMsg id 3017 "unary -a in place of -e is" bashism (TA_Unary id op _) | op `elem` [ "|++", "|--", "++|", "--|"] = warnMsg id 3018 $ filter (/= '|') op ++ " is" From 9605396bef40abdb830bc4c607c3736c007e3482 Mon Sep 17 00:00:00 2001 From: Max Ulidtko Date: Sun, 1 Oct 2023 21:23:25 +0200 Subject: [PATCH 623/763] Docs: describe fixes of PR #2837 in changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0338f7d..b9ce1fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,12 @@ - SC2324: Warn when x+=1 appends instead of increments - SC2325: Warn about multiple `!`s in dash/sh. - SC2326: Warn about `foo | ! bar` in bash/dash/sh. +- SC3012: Warn about lexicographic-compare bashism in test like in [ ] +- SC3013: Warn bashism `test _ -op/-nt/-ef _` like in [ ] +- SC3014: Warn bashism `test _ == _` like in [ ] +- SC3015: Warn bashism `test _ =~ _` like in [ ] +- SC3016: Warn bashism `test -v _` like in [ ] +- SC3017: Warn bashism `test -a _` like in [ ] ### Fixed - source statements with here docs now work correctly From 6a6d8e9fc488b3b20ab93da2915cccb42e208ddc Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 8 Oct 2023 18:52:05 -0700 Subject: [PATCH 624/763] Revert "Bump actions/checkout from 3 to 4" This reverts commit 410ec54617c86586cd6b5662ecc035898bd8aefa. --- .github/workflows/build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2ca94f2..a435cf4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,7 +15,7 @@ jobs: sudo apt-get install cabal-install - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v3 with: fetch-depth: 0 @@ -51,7 +51,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v3 - name: Download artifacts uses: actions/download-artifact@v3 @@ -74,7 +74,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v3 - name: Download artifacts uses: actions/download-artifact@v3 @@ -104,7 +104,7 @@ jobs: environment: Deploy steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v3 - name: Download artifacts uses: actions/download-artifact@v3 From 99a94421ab77397ae98e192a722ba7e61c103dec Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 8 Oct 2023 19:42:31 -0700 Subject: [PATCH 625/763] Manually install 'hub' dependency --- .github/workflows/build.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a435cf4..3e6fb27 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -103,6 +103,11 @@ jobs: runs-on: ubuntu-latest environment: Deploy steps: + - name: Install Dependencies + run: | + sudo apt-get update + sudo apt-get install hub + - name: Checkout repository uses: actions/checkout@v3 From dc2f388310c48eefe4613edf71a703f07d02ffa7 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sat, 14 Oct 2023 18:12:51 -0400 Subject: [PATCH 626/763] Adjust bounds to compile on 9.8 You'll need --allow-newer=fgl:deepseq for it to work too, until haskell/fgl#111 gets merged. --- ShellCheck.cabal | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ShellCheck.cabal b/ShellCheck.cabal index f09521f..76516db 100644 --- a/ShellCheck.cabal +++ b/ShellCheck.cabal @@ -46,13 +46,13 @@ library semigroups build-depends: -- The lower bounds are based on GHC 7.10.3 - -- The upper bounds are based on GHC 9.6.1 - aeson >= 1.4.0 && < 2.2, + -- The upper bounds are based on GHC 9.8.1 + aeson >= 1.4.0 && < 2.3, array >= 0.5.1 && < 0.6, base >= 4.8.0.0 && < 5, - bytestring >= 0.10.6 && < 0.12, - containers >= 0.5.6 && < 0.7, - deepseq >= 1.4.1 && < 1.5, + bytestring >= 0.10.6 && < 0.13, + containers >= 0.5.6 && < 0.8, + deepseq >= 1.4.1 && < 1.6, Diff >= 0.4.0 && < 0.5, fgl (>= 5.7.0 && < 5.8.1.0) || (>= 5.8.1.1 && < 5.9), filepath >= 1.4.0 && < 1.5, From 8b3c37aa36bcb74b09d71eba4ce0ce0141b8bd4f Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Mon, 16 Oct 2023 00:06:53 -0400 Subject: [PATCH 627/763] Use find instead of listToMaybe and filter --- src/ShellCheck/Analytics.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 7e8b510..3cebb24 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -3320,7 +3320,7 @@ checkReturnAgainstZero params token = isFirstCommandInFunction = fromMaybe False $ do let path = getPath (parentMap params) token - func <- listToMaybe $ filter isFunction path + func <- find isFunction path cmd <- getClosestCommand (parentMap params) token return $ getId cmd == getId (getFirstCommandInFunction func) From 4fd0615501b6909c8ca61ef080bb07b8e8de1301 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Mon, 16 Oct 2023 00:55:04 -0400 Subject: [PATCH 628/763] Stop using head in isLeadingNumberVar --- src/ShellCheck/Analytics.hs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 3cebb24..5f77f77 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -4378,9 +4378,8 @@ checkEqualsInCommand params originalToken = return $ isVariableName str isLeadingNumberVar s = - let lead = takeWhile (/= '=') s - in not (null lead) && isDigit (head lead) - && all isVariableChar lead && not (all isDigit lead) + case takeWhile (/= '=') s of + lead@(x:_) -> isDigit x && all isVariableChar lead && not (all isDigit lead) msg cmd leading (T_Literal litId s) = do -- There are many different cases, and the order of the branches matter. From 2a95bc6be3c9fe411e1ddb96e78b467ef2347bdb Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Mon, 16 Oct 2023 20:00:31 -0400 Subject: [PATCH 629/763] Switch to getLiteralStringDef to avoid an unnecessary fromJust --- src/ShellCheck/CFG.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/CFG.hs b/src/ShellCheck/CFG.hs index f882adc..2fe11e7 100644 --- a/src/ShellCheck/CFG.hs +++ b/src/ShellCheck/CFG.hs @@ -1058,7 +1058,7 @@ handleCommand cmd vars args literalCmd = do let id = getId t pre = [t] - literal = fromJust $ getLiteralStringExt (const $ Just "\0") t + literal = getLiteralStringDef "\0" t isKnown = '\0' `notElem` literal match = fmap head $ variableAssignRegex `matchRegex` literal name = fromMaybe literal match From 1aeab287e62ab562142fdb8a42e3f486f93216dd Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Fri, 3 Nov 2023 01:33:49 -0400 Subject: [PATCH 630/763] Add nil case that went missing in 4fd0615 --- src/ShellCheck/Analytics.hs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 5f77f77..dadf512 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -4380,6 +4380,7 @@ checkEqualsInCommand params originalToken = isLeadingNumberVar s = case takeWhile (/= '=') s of lead@(x:_) -> isDigit x && all isVariableChar lead && not (all isDigit lead) + [] -> False msg cmd leading (T_Literal litId s) = do -- There are many different cases, and the order of the branches matter. From be8e4b2b8aac4177ef31092397f1ea2cad8e66ad Mon Sep 17 00:00:00 2001 From: Grische Date: Sat, 25 Nov 2023 12:44:46 +0100 Subject: [PATCH 631/763] add basic busybox sh support --- CHANGELOG.md | 1 + src/ShellCheck/ASTLib.hs | 7 ++++--- src/ShellCheck/Analytics.hs | 4 ++-- src/ShellCheck/AnalyzerLib.hs | 4 ++-- src/ShellCheck/Data.hs | 2 ++ src/ShellCheck/Interface.hs | 5 ++--- src/ShellCheck/Parser.hs | 5 +++-- 7 files changed, 16 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b9ce1fb..897aa27 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ - SC3015: Warn bashism `test _ =~ _` like in [ ] - SC3016: Warn bashism `test -v _` like in [ ] - SC3017: Warn bashism `test -a _` like in [ ] +- Added support for busybox sh ### Fixed - source statements with here docs now work correctly diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 64fa762..cf55498 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -758,8 +758,8 @@ prop_executableFromShebang6 = executableFromShebang "/usr/bin/env --split-string prop_executableFromShebang7 = executableFromShebang "/usr/bin/env --split-string bash -x" == "bash" prop_executableFromShebang8 = executableFromShebang "/usr/bin/env --split-string foo=bar bash -x" == "bash" prop_executableFromShebang9 = executableFromShebang "/usr/bin/env foo=bar dash" == "dash" -prop_executableFromShebang10 = executableFromShebang "/bin/busybox sh" == "ash" -prop_executableFromShebang11 = executableFromShebang "/bin/busybox ash" == "ash" +prop_executableFromShebang10 = executableFromShebang "/bin/busybox sh" == "busybox sh" +prop_executableFromShebang11 = executableFromShebang "/bin/busybox ash" == "busybox ash" -- Get the shell executable from a string like '/usr/bin/env bash' executableFromShebang :: String -> String @@ -776,7 +776,8 @@ executableFromShebang = shellFor [x] -> basename x (first:second:args) | basename first == "busybox" -> case basename second of - "sh" -> "ash" -- busybox sh is ash + "sh" -> "busybox sh" + "ash" -> "busybox ash" x -> x (first:args) | basename first == "env" -> fromEnvArgs args diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index dadf512..5df2f35 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -646,10 +646,10 @@ prop_checkShebang9 = verifyNotTree checkShebang "# shellcheck shell=sh\ntrue" prop_checkShebang10 = verifyNotTree checkShebang "#!foo\n# shellcheck shell=sh ignore=SC2239\ntrue" prop_checkShebang11 = verifyTree checkShebang "#!/bin/sh/\ntrue" prop_checkShebang12 = verifyTree checkShebang "#!/bin/sh/ -xe\ntrue" -prop_checkShebang13 = verifyTree checkShebang "#!/bin/busybox sh" +prop_checkShebang13 = verifyNotTree checkShebang "#!/bin/busybox sh" prop_checkShebang14 = verifyNotTree checkShebang "#!/bin/busybox sh\n# shellcheck shell=sh\n" prop_checkShebang15 = verifyNotTree checkShebang "#!/bin/busybox sh\n# shellcheck shell=dash\n" -prop_checkShebang16 = verifyTree checkShebang "#!/bin/busybox ash" +prop_checkShebang16 = verifyNotTree checkShebang "#!/bin/busybox ash" prop_checkShebang17 = verifyNotTree checkShebang "#!/bin/busybox ash\n# shellcheck shell=dash\n" prop_checkShebang18 = verifyNotTree checkShebang "#!/bin/busybox ash\n# shellcheck shell=sh\n" checkShebang params (T_Annotation _ list t) = diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index ca928fd..d864e32 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -284,8 +284,8 @@ prop_determineShell7 = determineShellTest "#! /bin/ash" == Dash prop_determineShell8 = determineShellTest' (Just Ksh) "#!/bin/sh" == Sh prop_determineShell9 = determineShellTest "#!/bin/env -S dash -x" == Dash prop_determineShell10 = determineShellTest "#!/bin/env --split-string= dash -x" == Dash -prop_determineShell11 = determineShellTest "#!/bin/busybox sh" == Dash -- busybox sh is a specific shell, not posix sh -prop_determineShell12 = determineShellTest "#!/bin/busybox ash" == Dash +prop_determineShell11 = determineShellTest "#!/bin/busybox sh" == BusyboxSh -- busybox sh is a specific shell, not posix sh +prop_determineShell12 = determineShellTest "#!/bin/busybox ash" == BusyboxSh determineShellTest = determineShellTest' Nothing determineShellTest' fallbackShell = determineShell fallbackShell . fromJust . prRoot . pScript diff --git a/src/ShellCheck/Data.hs b/src/ShellCheck/Data.hs index 550ff87..6a87123 100644 --- a/src/ShellCheck/Data.hs +++ b/src/ShellCheck/Data.hs @@ -156,6 +156,8 @@ shellForExecutable name = "sh" -> return Sh "bash" -> return Bash "bats" -> return Bash + "busybox sh" -> return BusyboxSh + "busybox ash" -> return BusyboxSh "dash" -> return Dash "ash" -> return Dash -- There's also a warning for this. "ksh" -> return Ksh diff --git a/src/ShellCheck/Interface.hs b/src/ShellCheck/Interface.hs index 077212f..c574cee 100644 --- a/src/ShellCheck/Interface.hs +++ b/src/ShellCheck/Interface.hs @@ -28,7 +28,7 @@ module ShellCheck.Interface , AnalysisSpec(asScript, asShellType, asFallbackShell, asExecutionMode, asCheckSourced, asTokenPositions, asOptionalChecks) , AnalysisResult(arComments) , FormatterOptions(foColorOption, foWikiLinkCount) - , Shell(Ksh, Sh, Bash, Dash) + , Shell(Ksh, Sh, Bash, Dash, BusyboxSh) , ExecutionMode(Executed, Sourced) , ErrorMessage , Code @@ -221,7 +221,7 @@ newCheckDescription = CheckDescription { } -- Supporting data types -data Shell = Ksh | Sh | Bash | Dash deriving (Show, Eq) +data Shell = Ksh | Sh | Bash | Dash | BusyboxSh deriving (Show, Eq) data ExecutionMode = Executed | Sourced deriving (Show, Eq) type ErrorMessage = String @@ -335,4 +335,3 @@ mockedSystemInterface files = (newSystemInterface :: SystemInterface Identity) { mockRcFile rcfile mock = mock { siGetConfig = const . return $ Just (".shellcheckrc", rcfile) } - diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index ffc58e2..abd4d94 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -3349,8 +3349,8 @@ readScriptFile sourced = do verifyShebang pos s = do case isValidShell s of Just True -> return () - Just False -> parseProblemAt pos ErrorC 1071 "ShellCheck only supports sh/bash/dash/ksh scripts. Sorry!" - Nothing -> parseProblemAt pos ErrorC 1008 "This shebang was unrecognized. ShellCheck only supports sh/bash/dash/ksh. Add a 'shell' directive to specify." + Just False -> parseProblemAt pos ErrorC 1071 "ShellCheck only supports sh/bash/dash/ksh/'busybox sh' scripts. Sorry!" + Nothing -> parseProblemAt pos ErrorC 1008 "This shebang was unrecognized. ShellCheck only supports sh/bash/dash/ksh/'busybox sh'. Add a 'shell' directive to specify." isValidShell s = let good = null s || any (`isPrefixOf` s) goodShells @@ -3366,6 +3366,7 @@ readScriptFile sourced = do "sh", "ash", "dash", + "busybox sh", "bash", "bats", "ksh" From 1e1045e73e0c66a947390785e95d924203e837fc Mon Sep 17 00:00:00 2001 From: Grische Date: Sat, 25 Nov 2023 12:52:32 +0100 Subject: [PATCH 632/763] make busybox sh Dash-like --- src/ShellCheck/Analytics.hs | 6 +++++- src/ShellCheck/AnalyzerLib.hs | 4 ++++ src/ShellCheck/Checks/Commands.hs | 4 ++-- src/ShellCheck/Checks/ShellSupport.hs | 12 +++++++----- 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 5df2f35..108682a 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1204,6 +1204,7 @@ checkNumberComparisons params (TC_Binary id typ op lhs rhs) = do case shellType params of Sh -> return () -- These are unsupported and will be caught by bashism checks. Dash -> err id 2073 $ "Escape \\" ++ op ++ " to prevent it redirecting." + BusyboxSh -> err id 2073 $ "Escape \\" ++ op ++ " to prevent it redirecting." _ -> err id 2073 $ "Escape \\" ++ op ++ " to prevent it redirecting (or switch to [[ .. ]])." when (op `elem` arithmeticBinaryTestOps) $ do @@ -2782,6 +2783,7 @@ checkFunctionDeclarations params when (hasKeyword && hasParens) $ err id 2111 "ksh does not allow 'function' keyword and '()' at the same time." Dash -> forSh + BusyboxSh -> forSh Sh -> forSh where @@ -4044,7 +4046,8 @@ prop_checkModifiedArithmeticInRedirection3 = verifyNot checkModifiedArithmeticIn prop_checkModifiedArithmeticInRedirection4 = verify checkModifiedArithmeticInRedirection "cat <<< $((i++))" prop_checkModifiedArithmeticInRedirection5 = verify checkModifiedArithmeticInRedirection "cat << foo\n$((i++))\nfoo\n" prop_checkModifiedArithmeticInRedirection6 = verifyNot checkModifiedArithmeticInRedirection "#!/bin/dash\nls > $((i=i+1))" -checkModifiedArithmeticInRedirection params t = unless (shellType params == Dash) $ +prop_checkModifiedArithmeticInRedirection7 = verifyNot checkModifiedArithmeticInRedirection "#!/bin/busybox sh\ncat << foo\n$((i++))\nfoo\n" +checkModifiedArithmeticInRedirection params t = unless (shellType params == Dash || shellType params == BusyboxSh) $ case t of T_Redirecting _ redirs (T_SimpleCommand _ _ (_:_)) -> mapM_ checkRedirs redirs _ -> return () @@ -4356,6 +4359,7 @@ checkEqualsInCommand params originalToken = Bash -> errWithFix id 2277 "Use BASH_ARGV0 to assign to $0 in bash (or use [ ] to compare)." bashfix Ksh -> err id 2278 "$0 can't be assigned in Ksh (but it does reflect the current function)." Dash -> err id 2279 "$0 can't be assigned in Dash. This becomes a command name." + BusyboxSh -> err id 2279 "$0 can't be assigned in Busybox Ash. This becomes a command name." _ -> err id 2280 "$0 can't be assigned this way, and there is no portable alternative." leadingNumberMsg id = err id 2282 "Variable names can't start with numbers, so this is interpreted as a command." diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index d864e32..4990822 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -206,18 +206,21 @@ makeParameters spec = params case shellType params of Bash -> isOptionSet "lastpipe" root Dash -> False + BusyboxSh -> False Sh -> False Ksh -> True, hasInheritErrexit = case shellType params of Bash -> isOptionSet "inherit_errexit" root Dash -> True + BusyboxSh -> True Sh -> True Ksh -> False, hasPipefail = case shellType params of Bash -> isOptionSet "pipefail" root Dash -> True + BusyboxSh -> isOptionSet "pipefail" root Sh -> True Ksh -> isOptionSet "pipefail" root, shellTypeSpecified = isJust (asShellType spec) || isJust (asFallbackShell spec), @@ -899,6 +902,7 @@ isBashLike params = Bash -> True Ksh -> True Dash -> False + BusyboxSh -> False Sh -> False isTrueAssignmentSource c = diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 691836f..8be60a7 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -930,7 +930,7 @@ prop_checkTimedCommand2 = verify checkTimedCommand "#!/bin/dash\ntime ( foo; bar prop_checkTimedCommand3 = verifyNot checkTimedCommand "#!/bin/sh\ntime sleep 1" checkTimedCommand = CommandCheck (Exactly "time") f where f (T_SimpleCommand _ _ (c:args@(_:_))) = - whenShell [Sh, Dash] $ do + whenShell [Sh, Dash, BusyboxSh] $ do let cmd = last args -- "time" is parsed with a command as argument when (isPiped cmd) $ warn (getId c) 2176 "'time' is undefined for pipelines. time single stage or bash -c instead." @@ -954,7 +954,7 @@ checkTimedCommand = CommandCheck (Exactly "time") f where prop_checkLocalScope1 = verify checkLocalScope "local foo=3" prop_checkLocalScope2 = verifyNot checkLocalScope "f() { local foo=3; }" checkLocalScope = CommandCheck (Exactly "local") $ \t -> - whenShell [Bash, Dash] $ do -- Ksh allows it, Sh doesn't support local + whenShell [Bash, Dash, BusyboxSh] $ do -- Ksh allows it, Sh doesn't support local path <- getPathM t unless (any isFunctionLike path) $ err (getId $ getCommandTokenOrThis t) 2168 "'local' is only valid in functions." diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 1f012d2..7112b92 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -76,7 +76,7 @@ verifyNot c s = producesComments (testChecker c) s == Just False prop_checkForDecimals1 = verify checkForDecimals "((3.14*c))" prop_checkForDecimals2 = verify checkForDecimals "foo[1.2]=bar" prop_checkForDecimals3 = verifyNot checkForDecimals "declare -A foo; foo[1.2]=bar" -checkForDecimals = ForShell [Sh, Dash, Bash] f +checkForDecimals = ForShell [Sh, Dash, BusyboxSh, Bash] f where f t@(TA_Expansion id _) = sequence_ $ do str <- getLiteralString t @@ -196,14 +196,16 @@ prop_checkBashisms101 = verify checkBashisms "read" prop_checkBashisms102 = verifyNot checkBashisms "read -r foo" prop_checkBashisms103 = verifyNot checkBashisms "read foo" prop_checkBashisms104 = verifyNot checkBashisms "read ''" -checkBashisms = ForShell [Sh, Dash] $ \t -> do +prop_checkBashisms105 = verifyNot checkBashisms "#!/bin/busybox sh\nset -o pipefail" +checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do params <- ask kludge params t where -- This code was copy-pasted from Analytics where params was a variable kludge params = bashism where - isDash = shellType params == Dash + isBusyboxSh = shellType params == BusyboxSh + isDash = shellType params == Dash || isBusyboxSh warnMsg id code s = if isDash then err id code $ "In dash, " ++ s ++ " not supported." @@ -590,7 +592,7 @@ checkPS1Assignments = ForShell [Bash] f prop_checkMultipleBangs1 = verify checkMultipleBangs "! ! true" prop_checkMultipleBangs2 = verifyNot checkMultipleBangs "! true" -checkMultipleBangs = ForShell [Dash, Sh] f +checkMultipleBangs = ForShell [Dash, BusyboxSh, Sh] f where f token = case token of T_Banged id (T_Banged _ _) -> @@ -601,7 +603,7 @@ checkMultipleBangs = ForShell [Dash, Sh] f prop_checkBangAfterPipe1 = verify checkBangAfterPipe "true | ! true" prop_checkBangAfterPipe2 = verifyNot checkBangAfterPipe "true | ( ! true )" prop_checkBangAfterPipe3 = verifyNot checkBangAfterPipe "! ! true | true" -checkBangAfterPipe = ForShell [Dash, Sh, Bash] f +checkBangAfterPipe = ForShell [Dash, BusyboxSh, Sh, Bash] f where f token = case token of T_Pipeline _ _ cmds -> mapM_ check cmds From 00ffd2db33724c31d8857d278379f8140a3f5fce Mon Sep 17 00:00:00 2001 From: Grische Date: Sat, 25 Nov 2023 13:50:23 +0100 Subject: [PATCH 633/763] silence SC3010 for busybox sh --- src/ShellCheck/Checks/ShellSupport.hs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 7112b92..9ade1c1 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -197,6 +197,7 @@ prop_checkBashisms102 = verifyNot checkBashisms "read -r foo" prop_checkBashisms103 = verifyNot checkBashisms "read foo" prop_checkBashisms104 = verifyNot checkBashisms "read ''" prop_checkBashisms105 = verifyNot checkBashisms "#!/bin/busybox sh\nset -o pipefail" +prop_checkBashisms106 = verifyNot checkBashisms "#!/bin/busybox sh\nx=x\n[[ \"$x\" = \"$x\" ]]" checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do params <- ask kludge params t @@ -221,7 +222,8 @@ checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do bashism (T_DollarBracket id _) = warnMsg id 3007 "$[..] in place of $((..)) is" bashism (T_SelectIn id _ _ _) = warnMsg id 3008 "select loops are" bashism (T_BraceExpansion id _) = warnMsg id 3009 "brace expansion is" - bashism (T_Condition id DoubleBracket _) = warnMsg id 3010 "[[ ]] is" + bashism (T_Condition id DoubleBracket _) = + unless isBusyboxSh $ warnMsg id 3010 "[[ ]] is" bashism (T_HereString id _) = warnMsg id 3011 "here-strings are" bashism (TC_Binary id SingleBracket op _ _) | op `elem` [ "<", ">", "\\<", "\\>", "<=", ">=", "\\<=", "\\>="] = From 903421fb5dff31993c07b8b25eb0d46c388cfba0 Mon Sep 17 00:00:00 2001 From: Grische Date: Sat, 25 Nov 2023 13:53:13 +0100 Subject: [PATCH 634/763] silence SC3014 for busybox sh --- src/ShellCheck/Checks/ShellSupport.hs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 9ade1c1..e1a09dd 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -198,6 +198,7 @@ prop_checkBashisms103 = verifyNot checkBashisms "read foo" prop_checkBashisms104 = verifyNot checkBashisms "read ''" prop_checkBashisms105 = verifyNot checkBashisms "#!/bin/busybox sh\nset -o pipefail" prop_checkBashisms106 = verifyNot checkBashisms "#!/bin/busybox sh\nx=x\n[[ \"$x\" = \"$x\" ]]" +prop_checkBashisms107 = verifyNot checkBashisms "#!/bin/busybox sh\nx=x\n[ \"$x\" == \"$x\" ]" checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do params <- ask kludge params t @@ -238,9 +239,9 @@ checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do | op `elem` [ "-ot", "-nt", "-ef" ] = unless isDash $ warnMsg id 3013 $ op ++ " is" bashism (TC_Binary id SingleBracket "==" _ _) = - warnMsg id 3014 "== in place of = is" + unless isBusyboxSh $ warnMsg id 3014 "== in place of = is" bashism (T_SimpleCommand id _ [asStr -> Just "test", lhs, asStr -> Just "==", rhs]) = - warnMsg id 3014 "== in place of = is" + unless isBusyboxSh $ warnMsg id 3014 "== in place of = is" bashism (TC_Binary id SingleBracket "=~" _ _) = warnMsg id 3015 "=~ regex matching is" bashism (T_SimpleCommand id _ [asStr -> Just "test", lhs, asStr -> Just "=~", rhs]) = From ac63dc33c9b9b3576bc6e3f44049a9e6a53dfaea Mon Sep 17 00:00:00 2001 From: Grische Date: Sat, 25 Nov 2023 13:55:07 +0100 Subject: [PATCH 635/763] silence SC3020 for busybox sh --- src/ShellCheck/Checks/ShellSupport.hs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index e1a09dd..4409751 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -199,6 +199,7 @@ prop_checkBashisms104 = verifyNot checkBashisms "read ''" prop_checkBashisms105 = verifyNot checkBashisms "#!/bin/busybox sh\nset -o pipefail" prop_checkBashisms106 = verifyNot checkBashisms "#!/bin/busybox sh\nx=x\n[[ \"$x\" = \"$x\" ]]" prop_checkBashisms107 = verifyNot checkBashisms "#!/bin/busybox sh\nx=x\n[ \"$x\" == \"$x\" ]" +prop_checkBashisms108 = verifyNot checkBashisms "#!/bin/busybox sh\necho magic &> /dev/null" checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do params <- ask kludge params t @@ -258,7 +259,8 @@ checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do | op `elem` [ "|++", "|--", "++|", "--|"] = warnMsg id 3018 $ filter (/= '|') op ++ " is" bashism (TA_Binary id "**" _ _) = warnMsg id 3019 "exponentials are" - bashism (T_FdRedirect id "&" (T_IoFile _ (T_Greater _) _)) = warnMsg id 3020 "&> is" + bashism (T_FdRedirect id "&" (T_IoFile _ (T_Greater _) _)) = + unless isBusyboxSh $ warnMsg id 3020 "&> is" bashism (T_FdRedirect id "" (T_IoFile _ (T_GREATAND _) file)) = unless (all isDigit $ onlyLiteralString file) $ warnMsg id 3021 ">& filename (as opposed to >& fd) is" bashism (T_FdRedirect id ('{':_) _) = warnMsg id 3022 "named file descriptors are" From a3b8be82fe2b5d41837463bf81b1a92bb84e3835 Mon Sep 17 00:00:00 2001 From: Grische Date: Sat, 25 Nov 2023 13:58:49 +0100 Subject: [PATCH 636/763] silence SC3048 for busybox sh --- src/ShellCheck/Checks/ShellSupport.hs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 4409751..6ebdd70 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -200,6 +200,7 @@ prop_checkBashisms105 = verifyNot checkBashisms "#!/bin/busybox sh\nset -o pipef prop_checkBashisms106 = verifyNot checkBashisms "#!/bin/busybox sh\nx=x\n[[ \"$x\" = \"$x\" ]]" prop_checkBashisms107 = verifyNot checkBashisms "#!/bin/busybox sh\nx=x\n[ \"$x\" == \"$x\" ]" prop_checkBashisms108 = verifyNot checkBashisms "#!/bin/busybox sh\necho magic &> /dev/null" +prop_checkBashisms109 = verifyNot checkBashisms "#!/bin/busybox sh\ntrap stop EXIT SIGTERM" checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do params <- ask kludge params t @@ -399,7 +400,7 @@ checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do return $ do when (upper `elem` ["ERR", "DEBUG", "RETURN"]) $ warnMsg (getId token) 3047 $ "trapping " ++ str ++ " is" - when ("SIG" `isPrefixOf` upper) $ + when (not isBusyboxSh && "SIG" `isPrefixOf` upper) $ warnMsg (getId token) 3048 "prefixing signal names with 'SIG' is" when (not isDash && upper /= str) $ From ca255fe3263e53c7cb21618eac7d5e52e5c0a665 Mon Sep 17 00:00:00 2001 From: Grische Date: Sat, 25 Nov 2023 14:04:11 +0100 Subject: [PATCH 637/763] silence SC3046 and SC3051 for busybox sh --- src/ShellCheck/Checks/ShellSupport.hs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 6ebdd70..f4c93fc 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -201,6 +201,7 @@ prop_checkBashisms106 = verifyNot checkBashisms "#!/bin/busybox sh\nx=x\n[[ \"$x prop_checkBashisms107 = verifyNot checkBashisms "#!/bin/busybox sh\nx=x\n[ \"$x\" == \"$x\" ]" prop_checkBashisms108 = verifyNot checkBashisms "#!/bin/busybox sh\necho magic &> /dev/null" prop_checkBashisms109 = verifyNot checkBashisms "#!/bin/busybox sh\ntrap stop EXIT SIGTERM" +prop_checkBashisms110 = verifyNot checkBashisms "#!/bin/busybox sh\nsource /dev/null" checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do params <- ask kludge params t @@ -391,7 +392,8 @@ checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do (\x -> (not . null . snd $ x) && snd x `notElem` allowed) flags return . warnMsg (getId word) 3045 $ name ++ " -" ++ flag ++ " is" - when (name == "source") $ warnMsg id 3046 "'source' in place of '.' is" + when (name == "source" && not isBusyboxSh) $ + warnMsg id 3046 "'source' in place of '.' is" when (name == "trap") $ let check token = sequence_ $ do @@ -440,7 +442,9 @@ checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do ("wait", Just []) ] bashism t@(T_SourceCommand id src _) - | getCommandName src == Just "source" = warnMsg id 3051 "'source' in place of '.' is" + | getCommandName src == Just "source" = + unless isBusyboxSh $ + warnMsg id 3051 "'source' in place of '.' is" bashism (TA_Expansion _ (T_Literal id str : _)) | str `matches` radix = warnMsg id 3052 "arithmetic base conversion is" where From fdcce458c189848b76d3779e35581cd012af201d Mon Sep 17 00:00:00 2001 From: Grische Date: Sat, 25 Nov 2023 15:10:44 +0100 Subject: [PATCH 638/763] silence some shell expansions for busybox sh --- src/ShellCheck/Checks/ShellSupport.hs | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index f4c93fc..e652cb5 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -202,6 +202,15 @@ prop_checkBashisms107 = verifyNot checkBashisms "#!/bin/busybox sh\nx=x\n[ \"$x\ prop_checkBashisms108 = verifyNot checkBashisms "#!/bin/busybox sh\necho magic &> /dev/null" prop_checkBashisms109 = verifyNot checkBashisms "#!/bin/busybox sh\ntrap stop EXIT SIGTERM" prop_checkBashisms110 = verifyNot checkBashisms "#!/bin/busybox sh\nsource /dev/null" +prop_checkBashisms111 = verify checkBashisms "#!/bin/dash\nx='test'\n${x:0:3}" -- SC3057 +prop_checkBashisms112 = verifyNot checkBashisms "#!/bin/busybox sh\nx='test'\n${x:0:3}" -- SC3057 +prop_checkBashisms113 = verify checkBashisms "#!/bin/dash\nx='test'\n${x/st/xt}" -- SC3060 +prop_checkBashisms114 = verifyNot checkBashisms "#!/bin/busybox sh\nx='test'\n${x/st/xt}" -- SC3060 +prop_checkBashisms115 = verify checkBashisms "#!/bin/busybox sh\nx='test'\n${!x}" -- SC3053 +prop_checkBashisms116 = verify checkBashisms "#!/bin/busybox sh\nx='test'\n${x[1]}" -- SC3054 +prop_checkBashisms117 = verify checkBashisms "#!/bin/busybox sh\nx='test'\n${!x[@]}" -- SC3055 +prop_checkBashisms118 = verify checkBashisms "#!/bin/busybox sh\nxyz=1\n${!x*}" -- SC3056 +prop_checkBashisms119 = verify checkBashisms "#!/bin/busybox sh\nx='test'\n${x^^[t]}" -- SC3059 checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do params <- ask kludge params t @@ -282,7 +291,8 @@ checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do warnMsg id 3028 $ str ++ " is" bashism t@(T_DollarBraced id _ token) = do - mapM_ check expansion + unless isBusyboxSh $ mapM_ check simpleExpansions + mapM_ check advancedExpansions when (isBashVariable var) $ warnMsg id 3028 $ var ++ " is" where @@ -452,14 +462,16 @@ checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do bashism _ = return () varChars="_0-9a-zA-Z" - expansion = let re = mkRegex in [ + advancedExpansions = let re = mkRegex in [ (re $ "^![" ++ varChars ++ "]", 3053, "indirect expansion is"), (re $ "^[" ++ varChars ++ "]+\\[.*\\]$", 3054, "array references are"), (re $ "^![" ++ varChars ++ "]+\\[[*@]]$", 3055, "array key expansion is"), (re $ "^![" ++ varChars ++ "]+[*@]$", 3056, "name matching prefixes are"), + (re $ "^[" ++ varChars ++ "*@]+(\\[.*\\])?[,^]", 3059, "case modification is") + ] + simpleExpansions = let re = mkRegex in [ (re $ "^[" ++ varChars ++ "*@]+:[^-=?+]", 3057, "string indexing is"), (re $ "^([*@][%#]|#[@*])", 3058, "string operations on $@/$* are"), - (re $ "^[" ++ varChars ++ "*@]+(\\[.*\\])?[,^]", 3059, "case modification is"), (re $ "^[" ++ varChars ++ "*@]+(\\[.*\\])?/", 3060, "string replacement is") ] bashVars = [ From b6d4952e2e0602c894f6d9f28e7004c205cbcae7 Mon Sep 17 00:00:00 2001 From: Andreas Abel Date: Wed, 6 Dec 2023 18:41:53 +0100 Subject: [PATCH 639/763] Testsuite: report which module failed the tests This also fixes the problem that the testsuite threw `exitFailure` even when it succeeded (which I found inexplicable). Once this is published, the testsuite could be enabled in Stackage again. --- .gitignore | 1 + test/shellcheck.hs | 37 ++++++++++++++++++++----------------- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/.gitignore b/.gitignore index 6d5f1ae..cf373a8 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ cabal.config /parts/ /prime/ *.snap +/dist-newstyle/ diff --git a/test/shellcheck.hs b/test/shellcheck.hs index 1a272af..d5e056d 100644 --- a/test/shellcheck.hs +++ b/test/shellcheck.hs @@ -18,21 +18,24 @@ import qualified ShellCheck.Parser main = do putStrLn "Running ShellCheck tests..." - results <- sequence [ - ShellCheck.Analytics.runTests - ,ShellCheck.AnalyzerLib.runTests - ,ShellCheck.ASTLib.runTests - ,ShellCheck.CFG.runTests - ,ShellCheck.CFGAnalysis.runTests - ,ShellCheck.Checker.runTests - ,ShellCheck.Checks.Commands.runTests - ,ShellCheck.Checks.ControlFlow.runTests - ,ShellCheck.Checks.Custom.runTests - ,ShellCheck.Checks.ShellSupport.runTests - ,ShellCheck.Fixer.runTests - ,ShellCheck.Formatter.Diff.runTests - ,ShellCheck.Parser.runTests + failures <- filter (not . snd) <$> mapM sequenceA tests + if null failures then exitSuccess else do + putStrLn "Tests failed for the following module(s):" + mapM (putStrLn . ("- ShellCheck." ++) . fst) failures + exitFailure + where + tests = + [ ("Analytics" , ShellCheck.Analytics.runTests) + , ("AnalyzerLib" , ShellCheck.AnalyzerLib.runTests) + , ("ASTLib" , ShellCheck.ASTLib.runTests) + , ("CFG" , ShellCheck.CFG.runTests) + , ("CFGAnalysis" , ShellCheck.CFGAnalysis.runTests) + , ("Checker" , ShellCheck.Checker.runTests) + , ("Checks.Commands" , ShellCheck.Checks.Commands.runTests) + , ("Checks.ControlFlow" , ShellCheck.Checks.ControlFlow.runTests) + , ("Checks.Custom" , ShellCheck.Checks.Custom.runTests) + , ("Checks.ShellSupport", ShellCheck.Checks.ShellSupport.runTests) + , ("Fixer" , ShellCheck.Fixer.runTests) + , ("Formatter.Diff" , ShellCheck.Formatter.Diff.runTests) + , ("Parser" , ShellCheck.Parser.runTests) ] - if and results - then exitSuccess - else exitFailure From 74282b0a9319bdcc0e3a4a17e2db1969e1c73be2 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 10 Dec 2023 17:05:29 -0800 Subject: [PATCH 640/763] Recognize 'busybox' in --shell and directives. Add to doc texts. --- shellcheck.1.md | 3 ++- shellcheck.hs | 2 +- src/ShellCheck/Checks/ShellSupport.hs | 2 ++ src/ShellCheck/Data.hs | 1 + 4 files changed, 6 insertions(+), 2 deletions(-) diff --git a/shellcheck.1.md b/shellcheck.1.md index 9675e79..89f6d50 100644 --- a/shellcheck.1.md +++ b/shellcheck.1.md @@ -85,7 +85,8 @@ not warn at all, as `ksh` supports decimals in arithmetic contexts. **-s**\ *shell*,\ **--shell=***shell* -: Specify Bourne shell dialect. Valid values are *sh*, *bash*, *dash* and *ksh*. +: Specify Bourne shell dialect. Valid values are *sh*, *bash*, *dash*, *ksh*, + and *busybox*. The default is to deduce the shell from the file's `shell` directive, shebang, or `.bash/.bats/.dash/.ksh` extension, in that order. *sh* refers to POSIX `sh` (not the system's), and will warn of portability issues. diff --git a/shellcheck.hs b/shellcheck.hs index 6be9bb1..6f12238 100644 --- a/shellcheck.hs +++ b/shellcheck.hs @@ -115,7 +115,7 @@ options = [ "Specify path when looking for sourced files (\"SCRIPTDIR\" for script's dir)", Option "s" ["shell"] (ReqArg (Flag "shell") "SHELLNAME") - "Specify dialect (sh, bash, dash, ksh)", + "Specify dialect (sh, bash, dash, ksh, busybox)", Option "S" ["severity"] (ReqArg (Flag "severity") "SEVERITY") "Minimum severity of errors to consider (error, warning, info, style)", diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index e652cb5..d070497 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -211,6 +211,8 @@ prop_checkBashisms116 = verify checkBashisms "#!/bin/busybox sh\nx='test'\n${x[1 prop_checkBashisms117 = verify checkBashisms "#!/bin/busybox sh\nx='test'\n${!x[@]}" -- SC3055 prop_checkBashisms118 = verify checkBashisms "#!/bin/busybox sh\nxyz=1\n${!x*}" -- SC3056 prop_checkBashisms119 = verify checkBashisms "#!/bin/busybox sh\nx='test'\n${x^^[t]}" -- SC3059 +prop_checkBashisms120 = verify checkBashisms "#!/bin/sh\n[ x == y ]" +prop_checkBashisms121 = verifyNot checkBashisms "#!/bin/sh\n# shellcheck shell=busybox\n[ x == y ]" checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do params <- ask kludge params t diff --git a/src/ShellCheck/Data.hs b/src/ShellCheck/Data.hs index 6a87123..917142e 100644 --- a/src/ShellCheck/Data.hs +++ b/src/ShellCheck/Data.hs @@ -156,6 +156,7 @@ shellForExecutable name = "sh" -> return Sh "bash" -> return Bash "bats" -> return Bash + "busybox" -> return BusyboxSh -- Used for directives and --shell=busybox "busybox sh" -> return BusyboxSh "busybox ash" -> return BusyboxSh "dash" -> return Dash From f2729f73cbffde5ef332ce943bc07021302781ab Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 10 Dec 2023 17:57:33 -0800 Subject: [PATCH 641/763] Abuse STRIP to avoid crashes on unsupported AST nodes --- src/ShellCheck/CFG.hs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/CFG.hs b/src/ShellCheck/CFG.hs index 2fe11e7..0cd6326 100644 --- a/src/ShellCheck/CFG.hs +++ b/src/ShellCheck/CFG.hs @@ -887,7 +887,9 @@ build t = do T_Less _ -> none T_ParamSubSpecialChar _ _ -> none - x -> error ("Unimplemented: " ++ show x) + x -> do + error ("Unimplemented: " ++ show x) -- STRIP + none -- Still in `where` clause forInHelper id name words body = do From a9e7bf1950ed50bdfbc818710085c6414f8bf20e Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 10 Dec 2023 19:13:34 -0800 Subject: [PATCH 642/763] Reparse indices after attaching here docs (fixes #2846) --- src/ShellCheck/Checker.hs | 3 +++ src/ShellCheck/Parser.hs | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/ShellCheck/Checker.hs b/src/ShellCheck/Checker.hs index c79f90f..6c9166f 100644 --- a/src/ShellCheck/Checker.hs +++ b/src/ShellCheck/Checker.hs @@ -516,6 +516,9 @@ prop_hereDocsAreParsedWithoutTrailingLinefeed = 1044 `elem` result where result = check "cat << eof" +prop_hereDocsWillHaveParsedIndices = null result + where + result = check "#!/bin/bash\nmy_array=(a b)\ncat <> ./test\n $(( 1 + my_array[1] ))\nEOF" return [] runTests = $quickCheckAll diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index abd4d94..701010f 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -3339,7 +3339,8 @@ readScriptFile sourced = do verifyEof let script = T_Annotation annotationId annotations $ T_Script id shebang commands - reparseIndices script + userstate <- getState + reparseIndices $ reattachHereDocs script (hereDocMap userstate) else do many anyChar id <- endSpan start @@ -3487,8 +3488,7 @@ parseShell env name contents = do return newParseResult { prComments = map toPositionedComment $ nub $ parseNotes userstate ++ parseProblems state, prTokenPositions = Map.map startEndPosToPos (positionMap userstate), - prRoot = Just $ - reattachHereDocs script (hereDocMap userstate) + prRoot = Just script } Left err -> do let context = contextStack state From 4c1d9171b26074e52736f3a58635d69a18326d68 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Mon, 11 Dec 2023 15:08:39 -0500 Subject: [PATCH 643/763] Remove partial head function from src/ShellCheck/Formatter/TTY.hs --- src/ShellCheck/Formatter/TTY.hs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/ShellCheck/Formatter/TTY.hs b/src/ShellCheck/Formatter/TTY.hs index e28696c..e503639 100644 --- a/src/ShellCheck/Formatter/TTY.hs +++ b/src/ShellCheck/Formatter/TTY.hs @@ -31,9 +31,9 @@ import Data.Ord import Data.IORef import Data.List import Data.Maybe -import GHC.Exts import System.IO import System.Info +import qualified Data.List.NonEmpty as NE wikiLink = "https://www.shellcheck.net/wiki/" @@ -117,19 +117,19 @@ outputResult options ref result sys = do color <- getColorFunc $ foColorOption options let comments = crComments result appendComments ref comments (fromIntegral $ foWikiLinkCount options) - let fileGroups = groupWith sourceFile comments + let fileGroups = NE.groupWith sourceFile comments mapM_ (outputForFile color sys) fileGroups outputForFile color sys comments = do - let fileName = sourceFile (head comments) + let fileName = sourceFile (NE.head comments) result <- siReadFile sys (Just True) fileName let contents = either (const "") id result let fileLinesList = lines contents let lineCount = length fileLinesList let fileLines = listArray (1, lineCount) fileLinesList - let groups = groupWith lineNo comments + let groups = NE.groupWith lineNo comments forM_ groups $ \commentsForLine -> do - let lineNum = fromIntegral $ lineNo (head commentsForLine) + let lineNum = fromIntegral $ lineNo (NE.head commentsForLine) let line = if lineNum < 1 || lineNum > lineCount then "" else fileLines ! fromIntegral lineNum @@ -139,7 +139,7 @@ outputForFile color sys comments = do putStrLn (color "source" line) forM_ commentsForLine $ \c -> putStrLn $ color (severityText c) $ cuteIndent c putStrLn "" - showFixedString color commentsForLine (fromIntegral lineNum) fileLines + showFixedString color (toList commentsForLine) (fromIntegral lineNum) fileLines -- Pick out only the lines necessary to show a fix in action sliceFile :: Fix -> Array Int String -> (Fix, Array Int String) From e5208ccb50e3d10957c13f0b77d19936fa4842e1 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Mon, 11 Dec 2023 15:43:35 -0500 Subject: [PATCH 644/763] Remove partial head function from src/ShellCheck/Formatter/JSON1.hs --- src/ShellCheck/Formatter/JSON1.hs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/Formatter/JSON1.hs b/src/ShellCheck/Formatter/JSON1.hs index 2169bf6..b4dbe35 100644 --- a/src/ShellCheck/Formatter/JSON1.hs +++ b/src/ShellCheck/Formatter/JSON1.hs @@ -27,9 +27,9 @@ import Control.DeepSeq import Data.Aeson import Data.IORef import Data.Monoid -import GHC.Exts import System.IO import qualified Data.ByteString.Lazy.Char8 as BL +import qualified Data.List.NonEmpty as NE format :: IO Formatter format = do @@ -114,10 +114,10 @@ outputError file msg = hPutStrLn stderr $ file ++ ": " ++ msg collectResult ref cr sys = mapM_ f groups where comments = crComments cr - groups = groupWith sourceFile comments - f :: [PositionedComment] -> IO () + groups = NE.groupWith sourceFile comments + f :: NE.NonEmpty PositionedComment -> IO () f group = do - let filename = sourceFile (head group) + let filename = sourceFile (NE.head group) result <- siReadFile sys (Just True) filename let contents = either (const "") id result let comments' = makeNonVirtual comments contents From 5a961371a75baea7d04ce96bf8b85e548f566e1a Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Mon, 11 Dec 2023 15:55:29 -0500 Subject: [PATCH 645/763] Remove partial head function from src/ShellCheck/Formatter/GCC.hs --- src/ShellCheck/Formatter/GCC.hs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/ShellCheck/Formatter/GCC.hs b/src/ShellCheck/Formatter/GCC.hs index 5106e4c..b921753 100644 --- a/src/ShellCheck/Formatter/GCC.hs +++ b/src/ShellCheck/Formatter/GCC.hs @@ -23,8 +23,8 @@ import ShellCheck.Interface import ShellCheck.Formatter.Format import Data.List -import GHC.Exts import System.IO +import qualified Data.List.NonEmpty as NE format :: IO Formatter format = return Formatter { @@ -39,13 +39,13 @@ outputError file error = hPutStrLn stderr $ file ++ ": " ++ error outputAll cr sys = mapM_ f groups where comments = crComments cr - groups = groupWith sourceFile comments - f :: [PositionedComment] -> IO () + groups = NE.groupWith sourceFile comments + f :: NE.NonEmpty PositionedComment -> IO () f group = do - let filename = sourceFile (head group) + let filename = sourceFile (NE.head group) result <- siReadFile sys (Just True) filename let contents = either (const "") id result - outputResult filename contents group + outputResult filename contents (NE.toList group) outputResult filename contents warnings = do let comments = makeNonVirtual warnings contents From e5028481e24e6e95ee11c6fae2323fca80449700 Mon Sep 17 00:00:00 2001 From: slycordinator <68940237+slycordinator@users.noreply.github.com> Date: Thu, 14 Dec 2023 15:24:49 +0900 Subject: [PATCH 646/763] Add installation directions for winge ShellCheck is now available on winget, so we can add it to the installation methods. --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 9d98b9d..ca9b847 100644 --- a/README.md +++ b/README.md @@ -194,6 +194,12 @@ On Windows (via [chocolatey](https://chocolatey.org/packages/shellcheck)): C:\> choco install shellcheck ``` +Or Windows (via [winget](https://github.com/microsoft/winget-pkgs)): + +```cmd +C:\> winget install --id koalaman.shellcheck +``` + Or Windows (via [scoop](http://scoop.sh)): ```cmd From 09d04c4c9b7dc0c8b466a6976d901ef1fc5c52e5 Mon Sep 17 00:00:00 2001 From: Jens Petersen Date: Fri, 15 Dec 2023 22:40:48 +0800 Subject: [PATCH 647/763] .cabal: allow Diff-0.5 --- ShellCheck.cabal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ShellCheck.cabal b/ShellCheck.cabal index 76516db..a12f75e 100644 --- a/ShellCheck.cabal +++ b/ShellCheck.cabal @@ -53,7 +53,7 @@ library bytestring >= 0.10.6 && < 0.13, containers >= 0.5.6 && < 0.8, deepseq >= 1.4.1 && < 1.6, - Diff >= 0.4.0 && < 0.5, + Diff >= 0.4.0 && < 0.6, fgl (>= 5.7.0 && < 5.8.1.0) || (>= 5.8.1.1 && < 5.9), filepath >= 1.4.0 && < 1.5, mtl >= 2.2.2 && < 2.4, From a37803d2b873329f062788f5bcfd20fca8f45edc Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Mon, 18 Dec 2023 23:57:47 -0500 Subject: [PATCH 648/763] Remove partial head function from src/ShellCheck/Formatter/CheckStyle.hs --- src/ShellCheck/Formatter/CheckStyle.hs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/Formatter/CheckStyle.hs b/src/ShellCheck/Formatter/CheckStyle.hs index 6ad6c9c..3f898c3 100644 --- a/src/ShellCheck/Formatter/CheckStyle.hs +++ b/src/ShellCheck/Formatter/CheckStyle.hs @@ -24,8 +24,8 @@ import ShellCheck.Formatter.Format import Data.Char import Data.List -import GHC.Exts import System.IO +import qualified Data.List.NonEmpty as NE format :: IO Formatter format = return Formatter { @@ -45,12 +45,12 @@ outputResults cr sys = else mapM_ outputGroup fileGroups where comments = crComments cr - fileGroups = groupWith sourceFile comments + fileGroups = NE.groupWith sourceFile comments outputGroup group = do - let filename = sourceFile (head group) + let filename = sourceFile (NE.head group) result <- siReadFile sys (Just True) filename let contents = either (const "") id result - outputFile filename contents group + outputFile filename contents (NE.toList group) outputFile filename contents warnings = do let comments = makeNonVirtual warnings contents From f242922a2e76ad761f23ffba9040293811e7862a Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Tue, 19 Dec 2023 00:00:32 -0500 Subject: [PATCH 649/763] Use onlyLiteralString in more places --- src/ShellCheck/Analytics.hs | 6 +++--- src/ShellCheck/Checks/Commands.hs | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 108682a..b7844dd 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1443,14 +1443,14 @@ prop_checkConstantNullary5 = verify checkConstantNullary "[[ true ]]" prop_checkConstantNullary6 = verify checkConstantNullary "[ 1 ]" prop_checkConstantNullary7 = verify checkConstantNullary "[ false ]" checkConstantNullary _ (TC_Nullary _ _ t) | isConstant t = - case fromMaybe "" $ getLiteralString t of + case onlyLiteralString t of "false" -> err (getId t) 2158 "[ false ] is true. Remove the brackets." "0" -> err (getId t) 2159 "[ 0 ] is true. Use 'false' instead." "true" -> style (getId t) 2160 "Instead of '[ true ]', just use 'true'." "1" -> style (getId t) 2161 "Instead of '[ 1 ]', use 'true'." _ -> err (getId t) 2078 "This expression is constant. Did you forget a $ somewhere?" where - string = fromMaybe "" $ getLiteralString t + string = onlyLiteralString t checkConstantNullary _ _ = return () @@ -2276,7 +2276,7 @@ checkFunctionsUsedExternally params t = (Just str, t) -> do let name = basename str let args = skipOver t argv - let argStrings = map (\x -> (fromMaybe "" $ getLiteralString x, x)) args + let argStrings = map (\x -> (onlyLiteralString x, x)) args let candidates = getPotentialCommands name argStrings mapM_ (checkArg name (getId t)) candidates _ -> return () diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 8be60a7..86fda24 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -186,7 +186,7 @@ checkCommand map t@(T_SimpleCommand id cmdPrefix (cmd:rest)) = sequence_ $ do M.findWithDefault nullCheck (Basename $ basename name) map t else if name == "builtin" && not (null rest) then let t' = T_SimpleCommand id cmdPrefix rest - selectedBuiltin = fromMaybe "" $ getLiteralString . head $ rest + selectedBuiltin = onlyLiteralString $ head rest in M.findWithDefault nullCheck (Exactly selectedBuiltin) map t' else do M.findWithDefault nullCheck (Exactly name) map t @@ -299,7 +299,7 @@ checkExpr = CommandCheck (Basename "expr") f where "'expr' expects 3+ arguments but sees 1. Make sure each operator/operand is a separate argument, and escape <>&|." [first, second] | - (fromMaybe "" $ getLiteralString first) /= "length" + onlyLiteralString first /= "length" && not (willSplit first || willSplit second) -> do checkOp first warn (getId t) 2307 From c97abdb939a52beefa576d984b104eb89e7667d9 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Tue, 19 Dec 2023 00:41:12 -0500 Subject: [PATCH 650/763] Make HereDocPending only hold the relevant pieces of a T_HereDoc instead of an arbitrary Token --- src/ShellCheck/Parser.hs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 701010f..0e2fc6d 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -160,7 +160,7 @@ data Context = deriving (Show) data HereDocContext = - HereDocPending Token [Context] -- on linefeed, read this T_HereDoc + HereDocPending Id Dashed Quoted String [Context] -- on linefeed, read this T_HereDoc deriving (Show) data UserState = UserState { @@ -238,12 +238,12 @@ addToHereDocMap id list = do hereDocMap = Map.insert id list map } -addPendingHereDoc t = do +addPendingHereDoc id d q str = do state <- getState context <- getCurrentContexts let docs = pendingHereDocs state putState $ state { - pendingHereDocs = HereDocPending t context : docs + pendingHereDocs = HereDocPending id d q str context : docs } popPendingHereDocs = do @@ -1835,7 +1835,7 @@ readHereDoc = called "here document" $ do -- add empty tokens for now, read the rest in readPendingHereDocs let doc = T_HereDoc hid dashed quoted endToken [] - addPendingHereDoc doc + addPendingHereDoc hid dashed quoted endToken return doc where unquote :: String -> (Quoted, String) @@ -1856,7 +1856,7 @@ readPendingHereDocs = do docs <- popPendingHereDocs mapM_ readDoc docs where - readDoc (HereDocPending (T_HereDoc id dashed quoted endToken _) ctx) = + readDoc (HereDocPending id dashed quoted endToken ctx) = swapContext ctx $ do docStartPos <- getPosition From c1452e0d174fe6c4c0a3775d6ee430de1bd8ccda Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Tue, 19 Dec 2023 00:53:08 -0500 Subject: [PATCH 651/763] Remove unnecessary partiality from kludgeAwayQuotes --- src/ShellCheck/Parser.hs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 0e2fc6d..115de64 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -46,6 +46,7 @@ import Text.Parsec.Error import Text.Parsec.Pos import qualified Control.Monad.Reader as Mr import qualified Control.Monad.State as Ms +import qualified Data.List.NonEmpty as NE import qualified Data.Map.Strict as Map import Test.QuickCheck.All (quickCheckAll) @@ -2904,8 +2905,8 @@ readLetSuffix = many1 (readIoRedirect <|> try readLetExpression <|> readCmdWord) kludgeAwayQuotes :: String -> SourcePos -> (String, SourcePos) kludgeAwayQuotes s p = case s of - first:rest@(_:_) -> - let (last:backwards) = reverse rest + first:second:rest -> + let (last NE.:| backwards) = NE.reverse (second NE.:| rest) middle = reverse backwards in if first `elem` "'\"" && first == last From 208e38358e8c07688a29867235971136a8ed0092 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Tue, 19 Dec 2023 01:00:20 -0500 Subject: [PATCH 652/763] Use a list comprehension to remove partiality from notesForContext --- src/ShellCheck/Parser.hs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 115de64..04bdbc4 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -3507,13 +3507,11 @@ parseShell env name contents = do -- A final pass for ignoring parse errors after failed parsing isIgnored stack note = any (contextItemDisablesCode False (codeForParseNote note)) stack -notesForContext list = zipWith ($) [first, second] $ filter isName list +notesForContext list = zipWith ($) [first, second] [(pos, str) | ContextName pos str <- list] where - isName (ContextName _ _) = True - isName _ = False - first (ContextName pos str) = ParseNote pos pos ErrorC 1073 $ + first (pos, str) = ParseNote pos pos ErrorC 1073 $ "Couldn't parse this " ++ str ++ ". Fix to allow more checks." - second (ContextName pos str) = ParseNote pos pos InfoC 1009 $ + second (pos, str) = ParseNote pos pos InfoC 1009 $ "The mentioned syntax error was in this " ++ str ++ "." -- Go over all T_UnparsedIndex and reparse them as either arithmetic or text From 0c46b8b2d5dffcb01b5a4689f16b687bc5a5f84e Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Tue, 19 Dec 2023 01:49:04 -0500 Subject: [PATCH 653/763] Use NonEmpty to remove partiality from handleCommand --- src/ShellCheck/CFG.hs | 41 +++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/src/ShellCheck/CFG.hs b/src/ShellCheck/CFG.hs index 0cd6326..5476ad5 100644 --- a/src/ShellCheck/CFG.hs +++ b/src/ShellCheck/CFG.hs @@ -51,6 +51,7 @@ import Control.Monad.Identity import Data.Array.Unboxed import Data.Array.ST import Data.List hiding (map) +import qualified Data.List.NonEmpty as NE import Data.Maybe import qualified Data.Map as M import qualified Data.Set as S @@ -857,8 +858,8 @@ build t = do status <- newNodeRange (CFSetExitCode id) linkRange assignments status - T_SimpleCommand id vars list@(cmd:_) -> - handleCommand t vars list $ getUnquotedLiteral cmd + T_SimpleCommand id vars (cmd:args) -> + handleCommand t vars (cmd NE.:| args) $ getUnquotedLiteral cmd T_SingleQuoted _ _ -> none @@ -925,8 +926,8 @@ handleCommand cmd vars args literalCmd = do -- TODO: Handle assignments in declaring commands case literalCmd of - Just "exit" -> regularExpansion vars args $ handleExit - Just "return" -> regularExpansion vars args $ handleReturn + Just "exit" -> regularExpansion vars (NE.toList args) $ handleExit + Just "return" -> regularExpansion vars (NE.toList args) $ handleReturn Just "unset" -> regularExpansionWithStatus vars args $ handleUnset args Just "declare" -> handleDeclare args @@ -949,14 +950,14 @@ handleCommand cmd vars args literalCmd = do -- This will mostly behave like 'command' but ok Just "builtin" -> case args of - [_] -> regular - (_:newargs@(newcmd:_)) -> - handleCommand newcmd vars newargs $ getLiteralString newcmd + _ NE.:| [] -> regular + (_ NE.:| newcmd:newargs) -> + handleCommand newcmd vars (newcmd NE.:| newargs) $ getLiteralString newcmd Just "command" -> case args of - [_] -> regular - (_:newargs@(newcmd:_)) -> - handleOthers (getId newcmd) vars newargs $ getLiteralString newcmd + _ NE.:| [] -> regular + (_ NE.:| newcmd:newargs) -> + handleOthers (getId newcmd) vars (newcmd NE.:| newargs) $ getLiteralString newcmd _ -> regular where @@ -984,7 +985,7 @@ handleCommand cmd vars args literalCmd = do unreachable <- newNode CFUnreachable return $ Range ret unreachable - handleUnset (cmd:args) = do + handleUnset (cmd NE.:| args) = do case () of _ | "n" `elem` flagNames -> unsetWith CFUndefineNameref _ | "v" `elem` flagNames -> unsetWith CFUndefineVariable @@ -1003,7 +1004,7 @@ handleCommand cmd vars args literalCmd = do variableAssignRegex = mkRegex "^([_a-zA-Z][_a-zA-Z0-9]*)=" - handleDeclare (cmd:args) = do + handleDeclare (cmd NE.:| args) = do isFunc <- asks cfIsFunction -- This is a bit of a kludge: we don't have great support for things like -- 'declare -i x=$x' so do one round with declare x=$x, followed by declare -i x @@ -1092,7 +1093,7 @@ handleCommand cmd vars args literalCmd = do in concatMap (drop 1) plusses - handlePrintf (cmd:args) = + handlePrintf (cmd NE.:| args) = newNodeRange $ CFApplyEffects $ maybeToList findVar where findVar = do @@ -1101,7 +1102,7 @@ handleCommand cmd vars args literalCmd = do name <- getLiteralString arg return $ IdTagged (getId arg) $ CFWriteVariable name CFValueString - handleWait (cmd:args) = + handleWait (cmd NE.:| args) = newNodeRange $ CFApplyEffects $ maybeToList findVar where findVar = do @@ -1110,7 +1111,7 @@ handleCommand cmd vars args literalCmd = do name <- getLiteralString arg return $ IdTagged (getId arg) $ CFWriteVariable name CFValueInteger - handleMapfile (cmd:args) = + handleMapfile (cmd NE.:| args) = newNodeRange $ CFApplyEffects [findVar] where findVar = @@ -1130,7 +1131,7 @@ handleCommand cmd vars args literalCmd = do guard $ isVariableName name return (getId c, name) - handleRead (cmd:args) = newNodeRange $ CFApplyEffects main + handleRead (cmd NE.:| args) = newNodeRange $ CFApplyEffects main where main = fromMaybe fallback $ do flags <- getGnuOpts flagsForRead args @@ -1160,7 +1161,7 @@ handleCommand cmd vars args literalCmd = do in map (\(id, name) -> IdTagged id $ CFWriteVariable name value) namesOrDefault - handleDEFINE (cmd:args) = + handleDEFINE (cmd NE.:| args) = newNodeRange $ CFApplyEffects $ maybeToList findVar where findVar = do @@ -1170,7 +1171,7 @@ handleCommand cmd vars args literalCmd = do return $ IdTagged (getId name) $ CFWriteVariable str CFValueString handleOthers id vars args cmd = - regularExpansion vars args $ do + regularExpansion vars (NE.toList args) $ do exe <- newNodeRange $ CFExecuteCommand cmd status <- newNodeRange $ CFSetExitCode id linkRange exe status @@ -1189,8 +1190,8 @@ handleCommand cmd vars args literalCmd = do linkRanges $ [args] ++ assignments ++ [exe] ++ dropAssignments - regularExpansionWithStatus vars args@(cmd:_) p = do - initial <- regularExpansion vars args p + regularExpansionWithStatus vars args@(cmd NE.:| _) p = do + initial <- regularExpansion vars (NE.toList args) p status <- newNodeRange $ CFSetExitCode (getId cmd) linkRange initial status From eed0174e90a374ee497aef7d7649e022856e80e9 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Tue, 19 Dec 2023 02:06:45 -0500 Subject: [PATCH 654/763] Make "Unresolved scope in dependency" impossible --- src/ShellCheck/CFG.hs | 24 ++++++++++++------------ src/ShellCheck/CFGAnalysis.hs | 17 ++++++++--------- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/src/ShellCheck/CFG.hs b/src/ShellCheck/CFG.hs index 5476ad5..ed6a8f8 100644 --- a/src/ShellCheck/CFG.hs +++ b/src/ShellCheck/CFG.hs @@ -112,8 +112,8 @@ data CFEdge = -- Actions we track data CFEffect = - CFSetProps Scope String (S.Set CFVariableProp) - | CFUnsetProps Scope String (S.Set CFVariableProp) + CFSetProps (Maybe Scope) String (S.Set CFVariableProp) + | CFUnsetProps (Maybe Scope) String (S.Set CFVariableProp) | CFReadVariable String | CFWriteVariable String CFValue | CFWriteGlobal String CFValue @@ -579,7 +579,7 @@ build t = do T_Array _ list -> sequentially list - T_Assignment {} -> buildAssignment DefaultScope t + T_Assignment {} -> buildAssignment Nothing t T_Backgrounded id body -> do start <- newStructuralNode @@ -1031,9 +1031,9 @@ handleCommand cmd vars args literalCmd = do scope isFunc = case () of - _ | global -> GlobalScope - _ | isFunc -> LocalScope - _ -> DefaultScope + _ | global -> Just GlobalScope + _ | isFunc -> Just LocalScope + _ -> Nothing addedProps = S.fromList $ concat $ [ [ CFVPArray | array ], @@ -1178,7 +1178,7 @@ handleCommand cmd vars args literalCmd = do regularExpansion vars args p = do args <- sequentially args - assignments <- mapM (buildAssignment PrefixScope) vars + assignments <- mapM (buildAssignment (Just PrefixScope)) vars exe <- p dropAssignments <- if null vars @@ -1198,7 +1198,7 @@ handleCommand cmd vars args literalCmd = do none = newStructuralNode -data Scope = DefaultScope | GlobalScope | LocalScope | PrefixScope +data Scope = GlobalScope | LocalScope | PrefixScope deriving (Eq, Ord, Show, Generic, NFData) buildAssignment scope t = do @@ -1212,10 +1212,10 @@ buildAssignment scope t = do let valueType = if null indices then f id value else CFValueArray let scoper = case scope of - PrefixScope -> CFWritePrefix - LocalScope -> CFWriteLocal - GlobalScope -> CFWriteGlobal - DefaultScope -> CFWriteVariable + Just PrefixScope -> CFWritePrefix + Just LocalScope -> CFWriteLocal + Just GlobalScope -> CFWriteGlobal + Nothing -> CFWriteVariable write <- newNodeRange $ applySingle $ IdTagged id $ scoper var valueType linkRanges [expand, index, read, write] where diff --git a/src/ShellCheck/CFGAnalysis.hs b/src/ShellCheck/CFGAnalysis.hs index 3b4f957..16afa68 100644 --- a/src/ShellCheck/CFGAnalysis.hs +++ b/src/ShellCheck/CFGAnalysis.hs @@ -299,7 +299,6 @@ depsToState set = foldl insert newInternalState $ S.toList set PrefixScope -> (sPrefixValues, insertPrefix) LocalScope -> (sLocalValues, insertLocal) GlobalScope -> (sGlobalValues, insertGlobal) - DefaultScope -> error $ pleaseReport "Unresolved scope in dependency" alreadyExists = isJust $ vmLookup name $ mapToCheck state in @@ -1120,34 +1119,34 @@ transferEffect ctx effect = CFSetProps scope name props -> case scope of - DefaultScope -> do + Nothing -> do state <- readVariable ctx name writeVariable ctx name $ addProperties props state - GlobalScope -> do + Just GlobalScope -> do state <- readGlobal ctx name writeGlobal ctx name $ addProperties props state - LocalScope -> do + Just LocalScope -> do out <- readSTRef (cOutput ctx) state <- readLocal ctx name writeLocal ctx name $ addProperties props state - PrefixScope -> do + Just PrefixScope -> do -- Prefix values become local state <- readLocal ctx name writeLocal ctx name $ addProperties props state CFUnsetProps scope name props -> case scope of - DefaultScope -> do + Nothing -> do state <- readVariable ctx name writeVariable ctx name $ removeProperties props state - GlobalScope -> do + Just GlobalScope -> do state <- readGlobal ctx name writeGlobal ctx name $ removeProperties props state - LocalScope -> do + Just LocalScope -> do out <- readSTRef (cOutput ctx) state <- readLocal ctx name writeLocal ctx name $ removeProperties props state - PrefixScope -> do + Just PrefixScope -> do -- Prefix values become local state <- readLocal ctx name writeLocal ctx name $ removeProperties props state From a47a42cb45e68050b84122a4bf237502b642350b Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Tue, 19 Dec 2023 02:17:59 -0500 Subject: [PATCH 655/763] Remove unnecessary partiality from isAssignmentParamToCommand --- src/ShellCheck/AnalyzerLib.hs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 4990822..1d53a98 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -341,9 +341,9 @@ isQuoteFreeNode strict shell tree t = -- Is this node self-quoting in itself? isQuoteFreeElement t = case t of - T_Assignment {} -> assignmentIsQuoting t - T_FdRedirect {} -> True - _ -> False + T_Assignment id _ _ _ _ -> assignmentIsQuoting id + T_FdRedirect {} -> True + _ -> False -- Are any subnodes inherently self-quoting? isQuoteFreeContext t = @@ -353,7 +353,7 @@ isQuoteFreeNode strict shell tree t = TC_Binary _ DoubleBracket _ _ _ -> return True TA_Sequence {} -> return True T_Arithmetic {} -> return True - T_Assignment {} -> return $ assignmentIsQuoting t + T_Assignment id _ _ _ _ -> return $ assignmentIsQuoting id T_Redirecting {} -> return False T_DoubleQuoted _ _ -> return True T_DollarDoubleQuoted _ _ -> return True @@ -368,11 +368,11 @@ isQuoteFreeNode strict shell tree t = -- Check whether this assignment is self-quoting due to being a recognized -- assignment passed to a Declaration Utility. This will soon be required -- by POSIX: https://austingroupbugs.net/view.php?id=351 - assignmentIsQuoting t = shellParsesParamsAsAssignments || not (isAssignmentParamToCommand t) + assignmentIsQuoting id = shellParsesParamsAsAssignments || not (isAssignmentParamToCommand id) shellParsesParamsAsAssignments = shell /= Sh -- Is this assignment a parameter to a command like export/typeset/etc? - isAssignmentParamToCommand (T_Assignment id _ _ _ _) = + isAssignmentParamToCommand id = case Map.lookup id tree of Just (T_SimpleCommand _ _ (_:args)) -> id `elem` (map getId args) _ -> False From bfe4342697292fe25a9214e30a0770fe0237ec42 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Tue, 19 Dec 2023 02:30:48 -0500 Subject: [PATCH 656/763] Remove unnecessary partiality from check --- src/ShellCheck/Checks/Commands.hs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 86fda24..314c1e9 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -1006,7 +1006,7 @@ checkWhileGetoptsCase = CommandCheck (Exactly "getopts") f options <- getLiteralString arg1 getoptsVar <- getLiteralString name (T_WhileExpression _ _ body) <- findFirst whileLoop path - caseCmd@(T_CaseExpression _ var _) <- mapMaybe findCase body !!! 0 + T_CaseExpression id var list <- mapMaybe findCase body !!! 0 -- Make sure getopts name and case variable matches [T_DollarBraced _ _ bracedWord] <- return $ getWordParts var @@ -1016,11 +1016,11 @@ checkWhileGetoptsCase = CommandCheck (Exactly "getopts") f -- Make sure the variable isn't modified guard . not $ modifiesVariable params (T_BraceGroup (Id 0) body) getoptsVar - return $ check (getId arg1) (map (:[]) $ filter (/= ':') options) caseCmd + return $ check (getId arg1) (map (:[]) $ filter (/= ':') options) id list f _ = return () - check :: Id -> [String] -> Token -> Analysis - check optId opts (T_CaseExpression id _ list) = do + check :: Id -> [String] -> Id -> [(CaseType, [Token], [Token])] -> Analysis + check optId opts id list = do unless (Nothing `M.member` handledMap) $ do mapM_ (warnUnhandled optId id) $ catMaybes $ M.keys notHandled From f983d9ae93e5eda28297a93a43640aafafa5f46c Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Thu, 21 Dec 2023 13:35:22 -0500 Subject: [PATCH 657/763] Simplify functionMap and remove unnecessary partiality --- src/ShellCheck/Analytics.hs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index b7844dd..19ff51b 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2825,13 +2825,11 @@ checkUnpassedInFunctions params root = execWriter $ mapM_ warnForGroup referenceGroups where functionMap :: Map.Map String Token - functionMap = Map.fromList $ - map (\t@(T_Function _ _ _ name _) -> (name,t)) functions - functions = execWriter $ doAnalysis (tell . maybeToList . findFunction) root + functionMap = Map.fromList $ execWriter $ doAnalysis (tell . maybeToList . findFunction) root findFunction t@(T_Function id _ _ name body) | any (isPositionalReference t) flow && not (any isPositionalAssignment flow) - = return t + = return (name,t) where flow = getVariableFlow params body findFunction _ = Nothing From dab77b2c8d978534603000e5406e604a79e1b195 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Thu, 21 Dec 2023 13:48:47 -0500 Subject: [PATCH 658/763] Implement parseEnum in terms of lookup --- shellcheck.hs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/shellcheck.hs b/shellcheck.hs index 6f12238..00b699b 100644 --- a/shellcheck.hs +++ b/shellcheck.hs @@ -252,9 +252,9 @@ runFormatter sys format options files = do else SomeProblems parseEnum name value list = - case filter ((== value) . fst) list of - [(name, value)] -> return value - [] -> do + case lookup value list of + Just value -> return value + Nothing -> do printErr $ "Unknown value for --" ++ name ++ ". " ++ "Valid options are: " ++ (intercalate ", " $ map fst list) throwError SupportFailure From 3bd7df955bdd9f066d5d19a90712b81305c61c87 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Fri, 29 Dec 2023 14:18:42 -0500 Subject: [PATCH 659/763] Use a pattern match instead of null and head in checkCommand --- src/ShellCheck/Checks/Commands.hs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 314c1e9..429e786 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -20,6 +20,7 @@ {-# LANGUAGE TemplateHaskell #-} {-# LANGUAGE FlexibleContexts #-} {-# LANGUAGE MultiWayIf #-} +{-# LANGUAGE PatternGuards #-} -- This module contains checks that examine specific commands by name. module ShellCheck.Checks.Commands (checker, optionalChecks, ShellCheck.Checks.Commands.runTests) where @@ -181,16 +182,15 @@ checkCommand :: M.Map CommandName (Token -> Analysis) -> Token -> Analysis checkCommand map t@(T_SimpleCommand id cmdPrefix (cmd:rest)) = sequence_ $ do name <- getLiteralString cmd return $ - if '/' `elem` name - then - M.findWithDefault nullCheck (Basename $ basename name) map t - else if name == "builtin" && not (null rest) then - let t' = T_SimpleCommand id cmdPrefix rest - selectedBuiltin = onlyLiteralString $ head rest - in M.findWithDefault nullCheck (Exactly selectedBuiltin) map t' - else do - M.findWithDefault nullCheck (Exactly name) map t - M.findWithDefault nullCheck (Basename name) map t + if | '/' `elem` name -> + M.findWithDefault nullCheck (Basename $ basename name) map t + | name == "builtin", (h:_) <- rest -> + let t' = T_SimpleCommand id cmdPrefix rest + selectedBuiltin = onlyLiteralString h + in M.findWithDefault nullCheck (Exactly selectedBuiltin) map t' + | otherwise -> do + M.findWithDefault nullCheck (Exactly name) map t + M.findWithDefault nullCheck (Basename name) map t where basename = reverse . takeWhile (/= '/') . reverse From dedf932fe8b9dcf4f852d4289032e6d25e7e5d40 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sat, 30 Dec 2023 13:59:15 -0500 Subject: [PATCH 660/763] Use traverse instead of sequence and map --- src/ShellCheck/Analytics.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 19ff51b..df0b3e6 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -4936,7 +4936,7 @@ checkOverwrittenExitCode params t = guard . not $ S.null exitCodeIds let idToToken = idMap params - exitCodeTokens <- sequence $ map (\k -> Map.lookup k idToToken) $ S.toList exitCodeIds + exitCodeTokens <- traverse (\k -> Map.lookup k idToToken) $ S.toList exitCodeIds return $ do when (all isCondition exitCodeTokens && not (usedUnconditionally t exitCodeIds)) $ warn id 2319 "This $? refers to a condition, not a command. Assign to a variable to avoid it being overwritten." From 980e7d3ca8aa19977ca517b846e2d2cff4fb1c5d Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sat, 30 Dec 2023 14:49:26 -0500 Subject: [PATCH 661/763] Use <$> instead of >>= and return --- src/ShellCheck/CFG.hs | 2 +- src/ShellCheck/Parser.hs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/CFG.hs b/src/ShellCheck/CFG.hs index ed6a8f8..5d018c8 100644 --- a/src/ShellCheck/CFG.hs +++ b/src/ShellCheck/CFG.hs @@ -997,7 +997,7 @@ handleCommand cmd vars args literalCmd = do (names, flags) = partition (null . fst) pairs flagNames = map fst flags literalNames :: [(Token, String)] -- Literal names to unset, e.g. [(myfuncToken, "myfunc")] - literalNames = mapMaybe (\(_, t) -> getLiteralString t >>= (return . (,) t)) names + literalNames = mapMaybe (\(_, t) -> (,) t <$> getLiteralString t) names -- Apply a constructor like CFUndefineVariable to each literalName, and tag with its id unsetWith c = newNodeRange $ CFApplyEffects $ map (\(token, name) -> IdTagged (getId token) $ c name) literalNames diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 04bdbc4..37a9b86 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -1195,7 +1195,7 @@ readDollarBracedPart = readSingleQuoted <|> readDoubleQuoted <|> readDollarBracedLiteral = do start <- startSpan - vars <- (readBraceEscaped <|> (anyChar >>= \x -> return [x])) `reluctantlyTill1` bracedQuotable + vars <- (readBraceEscaped <|> ((\x -> [x]) <$> anyChar)) `reluctantlyTill1` bracedQuotable id <- endSpan start return $ T_Literal id $ concat vars @@ -1557,7 +1557,7 @@ readGenericLiteral endChars = do return $ concat strings readGenericLiteral1 endExp = do - strings <- (readGenericEscaped <|> (anyChar >>= \x -> return [x])) `reluctantlyTill1` endExp + strings <- (readGenericEscaped <|> ((\x -> [x]) <$> anyChar)) `reluctantlyTill1` endExp return $ concat strings readGenericEscaped = do @@ -2371,7 +2371,7 @@ readPipeSequence = do return $ T_Pipeline id pipes cmds where sepBy1WithSeparators p s = do - let elems = p >>= \x -> return ([x], []) + let elems = (\x -> ([x], [])) <$> p let seps = do separator <- s return $ \(a,b) (c,d) -> (a++c, b ++ d ++ [separator]) From ee41c780f4a587cd32c58deb8badc27dac2b6b10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Sun, 31 Dec 2023 10:47:40 +0800 Subject: [PATCH 662/763] Replace Atom reference with Pulsar Edit equivalent Since Microsoft acquired GitHub and discontinued Atom in 2022, the community started a fork at https://pulsar-edit.dev/. Linking to an archived repository under the Atom organization does not make sense anymore, so link to active Pulsar fork instead. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9d98b9d..b9aea08 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,7 @@ You can see ShellCheck suggestions directly in a variety of editors. * Sublime, through [SublimeLinter](https://github.com/SublimeLinter/SublimeLinter-shellcheck). -* Atom, through [Linter](https://github.com/AtomLinter/linter-shellcheck). +* Pulsar Edit (former Atom), through [linter-shellcheck-pulsar](https://github.com/pulsar-cooperative/linter-shellcheck-pulsar). * VSCode, through [vscode-shellcheck](https://github.com/timonwong/vscode-shellcheck). From e1ad06383425f46703a26aafc0d1bdc8ddc90a18 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 31 Dec 2023 01:59:53 -0500 Subject: [PATCH 663/763] Implement getPath in terms of unfoldr --- src/ShellCheck/ASTLib.hs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index cf55498..5b3ffd8 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -897,10 +897,9 @@ getUnmodifiedParameterExpansion t = _ -> Nothing --- A list of the element and all its parents up to the root node. -getPath tree t = t : - case Map.lookup (getId t) tree of - Nothing -> [] - Just parent -> getPath tree parent +getPath tree t = t : unfoldr go t + where + go s = (\x -> (x,x)) <$> Map.lookup (getId s) tree isClosingFileOp op = case op of From add49cda171058b921fb57a2b658511159858a1f Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 31 Dec 2023 02:12:58 -0500 Subject: [PATCH 664/763] Make getPath return a NonEmpty --- src/ShellCheck/ASTLib.hs | 5 ++- src/ShellCheck/Analytics.hs | 59 ++++++++++++++++--------------- src/ShellCheck/AnalyzerLib.hs | 11 +++--- src/ShellCheck/Checks/Commands.hs | 3 +- 4 files changed, 40 insertions(+), 38 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 5b3ffd8..aadff05 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -31,6 +31,7 @@ import Data.Functor import Data.Functor.Identity import Data.List import Data.Maybe +import qualified Data.List.NonEmpty as NE import qualified Data.Map as Map import Numeric (showHex) @@ -897,9 +898,7 @@ getUnmodifiedParameterExpansion t = _ -> Nothing --- A list of the element and all its parents up to the root node. -getPath tree t = t : unfoldr go t - where - go s = (\x -> (x,x)) <$> Map.lookup (getId s) tree +getPath tree = NE.unfoldr $ \t -> (t, Map.lookup (getId t) tree) isClosingFileOp op = case op of diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index df0b3e6..e80ed58 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -46,6 +46,7 @@ import Data.Maybe import Data.Ord import Data.Semigroup import Debug.Trace -- STRIP +import qualified Data.List.NonEmpty as NE import qualified Data.Map.Strict as Map import qualified Data.Set as S import Test.QuickCheck.All (forAllProperties) @@ -846,14 +847,14 @@ checkRedirectToSame params s@(T_Pipeline _ _ list) = getRedirs _ = [] special x = "/dev/" `isPrefixOf` concat (oversimplify x) isInput t = - case drop 1 $ getPath (parentMap params) t of + case NE.tail $ getPath (parentMap params) t of T_IoFile _ op _:_ -> case op of T_Less _ -> True _ -> False _ -> False isOutput t = - case drop 1 $ getPath (parentMap params) t of + case NE.tail $ getPath (parentMap params) t of T_IoFile _ op _:_ -> case op of T_Greater _ -> True @@ -887,7 +888,7 @@ checkShorthandIf params x@(T_OrIf _ (T_AndIf id _ _) (T_Pipeline _ _ t)) name <- getCommandBasename t return $ name `elem` ["echo", "exit", "return", "printf"]) isOk _ = False - inCondition = isCondition $ getPath (parentMap params) x + inCondition = isCondition $ NE.toList $ getPath (parentMap params) x checkShorthandIf _ _ = return () @@ -1087,7 +1088,7 @@ checkSingleQuotedVariables params t@(T_SingleQuoted id s) = return $ if name == "find" then getFindCommand cmd else if name == "git" then getGitCommand cmd else if name == "mumps" then getMumpsCommand cmd else name isProbablyOk = - any isOkAssignment (take 3 $ getPath parents t) + any isOkAssignment (NE.take 3 $ getPath parents t) || commandName `elem` [ "trap" ,"sh" @@ -1495,7 +1496,7 @@ checkArithmeticDeref params t@(TA_Expansion _ [T_DollarBraced id _ l]) = where isException [] = True isException s@(h:_) = any (`elem` "/.:#%?*@$-!+=^,") s || isDigit h - getWarning = fromMaybe noWarning . msum . map warningFor $ parents params t + getWarning = fromMaybe noWarning . msum . NE.map warningFor $ parents params t warningFor t = case t of T_Arithmetic {} -> return normalWarning @@ -1823,7 +1824,7 @@ checkInexplicablyUnquoted params (T_NormalWord id tokens) = mapM_ check (tails t T_Literal id s | not (quotesSingleThing a && quotesSingleThing b || s `elem` ["=", ":", "/"] - || isSpecial (getPath (parentMap params) trapped) + || isSpecial (NE.toList $ getPath (parentMap params) trapped) ) -> warnAboutLiteral id _ -> return () @@ -2041,7 +2042,7 @@ doVariableFlowAnalysis readFunc writeFunc empty flow = evalState ( -- from $foo=bar to foo=bar. This is not pretty but ok. quotesMayConflictWithSC2281 params t = case getPath (parentMap params) t of - _ : T_NormalWord parentId (me:T_Literal _ ('=':_):_) : T_SimpleCommand _ _ (cmd:_) : _ -> + _ NE.:| T_NormalWord parentId (me:T_Literal _ ('=':_):_) : T_SimpleCommand _ _ (cmd:_) : _ -> (getId t) == (getId me) && (parentId == getId cmd) _ -> False @@ -2652,7 +2653,7 @@ checkPrefixAssignmentReference params t@(T_DollarBraced id _ value) = check path where name = getBracedReference $ concat $ oversimplify value - path = getPath (parentMap params) t + path = NE.toList $ getPath (parentMap params) t idPath = map getId path check [] = return () @@ -2701,7 +2702,7 @@ checkCharRangeGlob p t@(T_Glob id str) | return $ isCommandMatch cmd (`elem` ["tr", "read"]) -- Check if this is a dereferencing context like [[ -v array[operandhere] ]] - isDereferenced = fromMaybe False . msum . map isDereferencingOp . getPath (parentMap p) + isDereferenced = fromMaybe False . msum . NE.map isDereferencingOp . getPath (parentMap p) isDereferencingOp t = case t of TC_Binary _ DoubleBracket str _ _ -> return $ isDereferencingBinaryOp str @@ -2764,7 +2765,7 @@ checkLoopKeywordScope params t | _ -> return () where name = getCommandName t - path = let p = getPath (parentMap params) t in filter relevant p + path = let p = getPath (parentMap params) t in NE.filter relevant p subshellType t = case leadType params t of NoneScope -> Nothing SubshellScope str -> return str @@ -3188,7 +3189,7 @@ checkUncheckedCdPushdPopd params root = | name `elem` ["cd", "pushd", "popd"] && not (isSafeDir t) && not (name `elem` ["pushd", "popd"] && ("n" `elem` map snd (getAllFlags t))) - && not (isCondition $ getPath (parentMap params) t) = + && not (isCondition $ NE.toList $ getPath (parentMap params) t) = warnWithFix (getId t) 2164 ("Use '" ++ name ++ " ... || exit' or '" ++ name ++ " ... || return' in case " ++ name ++ " fails.") (fixWith [replaceEnd (getId t) params 0 " || exit"]) @@ -3217,7 +3218,7 @@ checkLoopVariableReassignment params token = return $ do warn (getId token) 2165 "This nested loop overrides the index variable of its parent." warn (getId next) 2167 "This parent loop has its index variable overridden." - path = drop 1 $ getPath (parentMap params) token + path = NE.tail $ getPath (parentMap params) token loopVariable :: Token -> Maybe String loopVariable t = case t of @@ -3290,17 +3291,17 @@ checkReturnAgainstZero params token = -- We don't want to warn about composite expressions like -- [[ $? -eq 0 || $? -eq 4 ]] since these can be annoying to rewrite. isOnlyTestInCommand t = - case getPath (parentMap params) t of - _:(T_Condition {}):_ -> True - _:(T_Arithmetic {}):_ -> True - _:(TA_Sequence _ [_]):(T_Arithmetic {}):_ -> True + case NE.tail $ getPath (parentMap params) t of + (T_Condition {}):_ -> True + (T_Arithmetic {}):_ -> True + (TA_Sequence _ [_]):(T_Arithmetic {}):_ -> True -- Some negations and groupings are also fine - _:next@(TC_Unary _ _ "!" _):_ -> isOnlyTestInCommand next - _:next@(TA_Unary _ "!" _):_ -> isOnlyTestInCommand next - _:next@(TC_Group {}):_ -> isOnlyTestInCommand next - _:next@(TA_Sequence _ [_]):_ -> isOnlyTestInCommand next - _:next@(TA_Parentesis _ _):_ -> isOnlyTestInCommand next + next@(TC_Unary _ _ "!" _):_ -> isOnlyTestInCommand next + next@(TA_Unary _ "!" _):_ -> isOnlyTestInCommand next + next@(TC_Group {}):_ -> isOnlyTestInCommand next + next@(TA_Sequence _ [_]):_ -> isOnlyTestInCommand next + next@(TA_Parentesis _ _):_ -> isOnlyTestInCommand next _ -> False -- TODO: Do better $? tracking and filter on whether @@ -3365,7 +3366,7 @@ checkRedirectedNowhere params token = _ -> return () where isInExpansion t = - case drop 1 $ getPath (parentMap params) t of + case NE.tail $ getPath (parentMap params) t of T_DollarExpansion _ [_] : _ -> True T_Backticked _ [_] : _ -> True t@T_Annotation {} : _ -> isInExpansion t @@ -3839,7 +3840,7 @@ checkSubshelledTests params t = isFunctionBody path = case path of - (_:f:_) -> isFunction f + (_ NE.:| f:_) -> isFunction f _ -> False isTestStructure t = @@ -3866,7 +3867,7 @@ checkSubshelledTests params t = -- This technically also triggers for `if true; then ( test ); fi` -- but it's still a valid suggestion. isCompoundCondition chain = - case dropWhile skippable (drop 1 chain) of + case dropWhile skippable (NE.tail chain) of T_IfExpression {} : _ -> True T_WhileExpression {} : _ -> True T_UntilExpression {} : _ -> True @@ -4005,7 +4006,7 @@ checkUselessBang params t = when (hasSetE params) $ mapM_ check (getNonReturning where check t = case t of - T_Banged id cmd | not $ isCondition (getPath (parentMap params) t) -> + T_Banged id cmd | not $ isCondition (NE.toList $ getPath (parentMap params) t) -> addComment $ makeCommentWithFix InfoC id 2251 "This ! is not on a condition and skips errexit. Use `&& exit 1` instead, or make sure $? is checked." (fixWith [replaceStart id params 1 "", replaceEnd (getId cmd) params 0 " && exit 1"]) @@ -4029,7 +4030,7 @@ checkUselessBang params t = when (hasSetE params) $ mapM_ check (getNonReturning isFunctionBody t = case getPath (parentMap params) t of - _:T_Function {}:_-> True + _ NE.:| T_Function {}:_-> True _ -> False dropLast t = @@ -4627,7 +4628,7 @@ checkArrayValueUsedAsIndex params _ = -- Is this one of the 'for' arrays? (loopWord, _) <- find ((==arrayName) . snd) arrays -- Are we still in this loop? - guard $ getId loop `elem` map getId (getPath parents t) + guard $ getId loop `elem` NE.map getId (getPath parents t) return [ makeComment WarningC (getId loopWord) 2302 "This loops over values. To loop over keys, use \"${!array[@]}\".", makeComment WarningC (getId arrayRef) 2303 $ (e4m name) ++ " is an array value, not a key. Use directly or loop over keys instead." @@ -4709,7 +4710,7 @@ checkSetESuppressed params t = literalArg <- getUnquotedLiteral cmd Map.lookup literalArg functions_ - checkCmd cmd = go $ getPath (parentMap params) cmd + checkCmd cmd = go $ NE.toList $ getPath (parentMap params) cmd where go (child:parent:rest) = do case parent of @@ -4855,7 +4856,7 @@ checkExtraMaskedReturns params t = basename <- getCommandBasename t return $ basename == "time" - parentChildPairs t = go $ parents params t + parentChildPairs t = go $ NE.toList $ parents params t where go (child:parent:rest) = (parent, child):go (parent:rest) go _ = [] diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 1d53a98..21123d4 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -41,6 +41,7 @@ import Data.Char import Data.List import Data.Maybe import Data.Semigroup +import qualified Data.List.NonEmpty as NE import qualified Data.Map as Map import Test.QuickCheck.All (forAllProperties) @@ -336,7 +337,7 @@ isQuoteFree = isQuoteFreeNode False isQuoteFreeNode strict shell tree t = isQuoteFreeElement t || - (fromMaybe False $ msum $ map isQuoteFreeContext $ drop 1 $ getPath tree t) + (fromMaybe False $ msum $ map isQuoteFreeContext $ NE.tail $ getPath tree t) where -- Is this node self-quoting in itself? isQuoteFreeElement t = @@ -398,7 +399,7 @@ isParamTo tree cmd = -- Get the parent command (T_Redirecting) of a Token, if any. getClosestCommand :: Map.Map Id Token -> Token -> Maybe Token getClosestCommand tree t = - findFirst findCommand $ getPath tree t + findFirst findCommand $ NE.toList $ getPath tree t where findCommand t = case t of @@ -412,7 +413,7 @@ getClosestCommandM t = do return $ getClosestCommand (parentMap params) t -- Is the token used as a command name (the first word in a T_SimpleCommand)? -usedAsCommandName tree token = go (getId token) (tail $ getPath tree token) +usedAsCommandName tree token = go (getId token) (NE.tail $ getPath tree token) where go currentId (T_NormalWord id [word]:rest) | currentId == getId word = go id rest @@ -429,7 +430,7 @@ getPathM t = do return $ getPath (parentMap params) t isParentOf tree parent child = - elem (getId parent) . map getId $ getPath tree child + elem (getId parent) . NE.map getId $ getPath tree child parents params = getPath (parentMap params) @@ -813,7 +814,7 @@ getReferencedVariables parents t = return (context, token, getBracedReference str) isArithmeticAssignment t = case getPath parents t of - this: TA_Assignment _ "=" lhs _ :_ -> lhs == t + this NE.:| TA_Assignment _ "=" lhs _ :_ -> lhs == t _ -> False isDereferencingBinaryOp = (`elem` ["-eq", "-ne", "-lt", "-le", "-gt", "-ge"]) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 429e786..c4ffd87 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -43,6 +43,7 @@ import Data.Functor.Identity import qualified Data.Graph.Inductive.Graph as G import Data.List import Data.Maybe +import qualified Data.List.NonEmpty as NE import qualified Data.Map.Strict as M import qualified Data.Set as S import Test.QuickCheck.All (forAllProperties) @@ -1005,7 +1006,7 @@ checkWhileGetoptsCase = CommandCheck (Exactly "getopts") f sequence_ $ do options <- getLiteralString arg1 getoptsVar <- getLiteralString name - (T_WhileExpression _ _ body) <- findFirst whileLoop path + (T_WhileExpression _ _ body) <- findFirst whileLoop (NE.toList path) T_CaseExpression id var list <- mapMaybe findCase body !!! 0 -- Make sure getopts name and case variable matches From 71c0fcb737e94d7f2aa65e0540ddc2554f63bdd7 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 31 Dec 2023 02:27:52 -0500 Subject: [PATCH 665/763] Manually fuse elem and map in isParentOf --- src/ShellCheck/AnalyzerLib.hs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 21123d4..944b12d 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -430,7 +430,9 @@ getPathM t = do return $ getPath (parentMap params) t isParentOf tree parent child = - elem (getId parent) . NE.map getId $ getPath tree child + any (\t -> parentId == getId t) (getPath tree child) + where + parentId = getId parent parents params = getPath (parentMap params) From 6e5b5401c6e593c82ce11ab15b532fe3a9c07d3b Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 31 Dec 2023 02:31:07 -0500 Subject: [PATCH 666/763] Manually fuse elem and map in checkArrayValueUsedAsIndex --- src/ShellCheck/Analytics.hs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index e80ed58..9926462 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -4628,7 +4628,8 @@ checkArrayValueUsedAsIndex params _ = -- Is this one of the 'for' arrays? (loopWord, _) <- find ((==arrayName) . snd) arrays -- Are we still in this loop? - guard $ getId loop `elem` NE.map getId (getPath parents t) + let loopId = getId loop + guard $ any (\t -> loopId == getId t) (getPath parents t) return [ makeComment WarningC (getId loopWord) 2302 "This loops over values. To loop over keys, use \"${!array[@]}\".", makeComment WarningC (getId arrayRef) 2303 $ (e4m name) ++ " is an array value, not a key. Use directly or loop over keys instead." From a786f996a176555ca5c09866dcb785cf7fd9323d Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 31 Dec 2023 15:55:06 -0500 Subject: [PATCH 667/763] Replace !!! with pattern-matching where it's easy --- src/ShellCheck/Analytics.hs | 14 +++++--------- src/ShellCheck/Checks/Commands.hs | 3 +-- src/ShellCheck/Checks/ShellSupport.hs | 5 ++--- 3 files changed, 8 insertions(+), 14 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 9926462..2fb3185 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -468,9 +468,8 @@ checkAssignAteCommand _ (T_SimpleCommand id [T_Assignment _ _ _ _ assignmentTerm where isCommonCommand (Just s) = s `elem` commonCommands isCommonCommand _ = False - firstWordIsArg list = fromMaybe False $ do - head <- list !!! 0 - return $ isGlob head || isUnquotedFlag head + firstWordIsArg (head:_) = isGlob head || isUnquotedFlag head + firstWordIsArg [] = False checkAssignAteCommand _ _ = return () @@ -491,9 +490,7 @@ prop_checkWrongArit2 = verify checkWrongArithmeticAssignment "n=2; i=n*2" checkWrongArithmeticAssignment params (T_SimpleCommand id [T_Assignment _ _ _ _ val] []) = sequence_ $ do str <- getNormalString val - match <- matchRegex regex str - var <- match !!! 0 - op <- match !!! 1 + var:op:_ <- matchRegex regex str Map.lookup var references return . warn (getId val) 2100 $ "Use $((..)) for arithmetics, e.g. i=$((i " ++ op ++ " 2))" @@ -1460,9 +1457,8 @@ prop_checkForDecimals2 = verify checkForDecimals "foo[1.2]=bar" prop_checkForDecimals3 = verifyNot checkForDecimals "declare -A foo; foo[1.2]=bar" checkForDecimals params t@(TA_Expansion id _) = sequence_ $ do guard $ not (hasFloatingPoint params) - str <- getLiteralString t - first <- str !!! 0 - guard $ isDigit first && '.' `elem` str + first:rest <- getLiteralString t + guard $ isDigit first && '.' `elem` rest return $ err id 2079 "(( )) doesn't support decimals. Use bc or awk." checkForDecimals _ _ = return () diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index c4ffd87..97c9088 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -1237,8 +1237,7 @@ checkSudoArgs = CommandCheck (Basename "sudo") f where f t = sequence_ $ do opts <- parseOpts $ arguments t - let nonFlags = [x | ("",(x, _)) <- opts] - commandArg <- nonFlags !!! 0 + (_,(commandArg, _)) <- find (null . fst) opts command <- getLiteralString commandArg guard $ command `elem` builtins return $ warn (getId t) 2232 $ "Can't use sudo with builtins like " ++ command ++ ". Did you want sudo sh -c .. instead?" diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index d070497..cab0546 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -79,9 +79,8 @@ prop_checkForDecimals3 = verifyNot checkForDecimals "declare -A foo; foo[1.2]=ba checkForDecimals = ForShell [Sh, Dash, BusyboxSh, Bash] f where f t@(TA_Expansion id _) = sequence_ $ do - str <- getLiteralString t - first <- str !!! 0 - guard $ isDigit first && '.' `elem` str + first:rest <- getLiteralString t + guard $ isDigit first && '.' `elem` rest return $ err id 2079 "(( )) doesn't support decimals. Use bc or awk." f _ = return () From 10afe83ce32c575cace720b51516092a44d39cf2 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 31 Dec 2023 16:23:45 -0500 Subject: [PATCH 668/763] Use getLiteralStringDef instead of rebuilding it with fromJust --- src/ShellCheck/Analytics.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 2fb3185..d030812 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -4509,7 +4509,7 @@ prop_checkCommandWithTrailingSymbol9 = verifyNot checkCommandWithTrailingSymbol checkCommandWithTrailingSymbol _ t = case t of T_SimpleCommand _ _ (cmd:_) -> - let str = fromJust $ getLiteralStringExt (\_ -> Just "x") cmd + let str = getLiteralStringDef "x" cmd last = lastOrDefault 'x' str in case str of From 6c81505870d84747c7614deebc2770f38c3e4c29 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 31 Dec 2023 16:26:03 -0500 Subject: [PATCH 669/763] Use a pattern guard instead of fromJust in checkLoopKeywordScope --- src/ShellCheck/Analytics.hs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index d030812..6daf614 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -19,6 +19,7 @@ -} {-# LANGUAGE TemplateHaskell #-} {-# LANGUAGE FlexibleContexts #-} +{-# LANGUAGE PatternGuards #-} module ShellCheck.Analytics (checker, optionalChecks, ShellCheck.Analytics.runTests) where import ShellCheck.AST @@ -2749,18 +2750,17 @@ prop_checkLoopKeywordScope5 = verify checkLoopKeywordScope "if true; then break; prop_checkLoopKeywordScope6 = verify checkLoopKeywordScope "while true; do true | { break; }; done" prop_checkLoopKeywordScope7 = verifyNot checkLoopKeywordScope "#!/bin/ksh\nwhile true; do true | { break; }; done" checkLoopKeywordScope params t | - name `elem` map Just ["continue", "break"] = + Just name <- getCommandName t, name `elem` ["continue", "break"] = if not $ any isLoop path then if any isFunction $ take 1 path -- breaking at a source/function invocation is an abomination. Let's ignore it. - then err (getId t) 2104 $ "In functions, use return instead of " ++ fromJust name ++ "." - else err (getId t) 2105 $ fromJust name ++ " is only valid in loops." + then err (getId t) 2104 $ "In functions, use return instead of " ++ name ++ "." + else err (getId t) 2105 $ name ++ " is only valid in loops." else case map subshellType $ filter (not . isFunction) path of Just str:_ -> warn (getId t) 2106 $ "This only exits the subshell caused by the " ++ str ++ "." _ -> return () where - name = getCommandName t path = let p = getPath (parentMap params) t in NE.filter relevant p subshellType t = case leadType params t of NoneScope -> Nothing From 3f40b688eee018f49b0a2ed7d8805987eb4118cd Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 31 Dec 2023 16:33:34 -0500 Subject: [PATCH 670/763] Simplify getStringFromParsec --- src/ShellCheck/Parser.hs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 37a9b86..130d956 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -3456,9 +3456,8 @@ makeErrorFor parsecError = pos = errorPos parsecError getStringFromParsec errors = - case map f errors of - r -> unwords (take 1 $ catMaybes $ reverse r) ++ - " Fix any mentioned problems and try again." + headOrDefault "" (mapMaybe f $ reverse errors) ++ + " Fix any mentioned problems and try again." where f err = case err of From a6984cddb0fbead307dce4bd98a66f8225ea2888 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 31 Dec 2023 16:40:18 -0500 Subject: [PATCH 671/763] Switch then and else to remove a not --- src/ShellCheck/Analytics.hs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 6daf614..ae29762 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2751,15 +2751,15 @@ prop_checkLoopKeywordScope6 = verify checkLoopKeywordScope "while true; do true prop_checkLoopKeywordScope7 = verifyNot checkLoopKeywordScope "#!/bin/ksh\nwhile true; do true | { break; }; done" checkLoopKeywordScope params t | Just name <- getCommandName t, name `elem` ["continue", "break"] = - if not $ any isLoop path - then if any isFunction $ take 1 path - -- breaking at a source/function invocation is an abomination. Let's ignore it. - then err (getId t) 2104 $ "In functions, use return instead of " ++ name ++ "." - else err (getId t) 2105 $ name ++ " is only valid in loops." - else case map subshellType $ filter (not . isFunction) path of + if any isLoop path + then case map subshellType $ filter (not . isFunction) path of Just str:_ -> warn (getId t) 2106 $ "This only exits the subshell caused by the " ++ str ++ "." _ -> return () + else if any isFunction $ take 1 path + -- breaking at a source/function invocation is an abomination. Let's ignore it. + then err (getId t) 2104 $ "In functions, use return instead of " ++ name ++ "." + else err (getId t) 2105 $ name ++ " is only valid in loops." where path = let p = getPath (parentMap params) t in NE.filter relevant p subshellType t = case leadType params t of From 71889c139aac3ce3597349878c746c774b329882 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 31 Dec 2023 16:44:21 -0500 Subject: [PATCH 672/763] Use a case expression instead of any and take 1 --- src/ShellCheck/Analytics.hs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index ae29762..ca679bb 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2756,10 +2756,10 @@ checkLoopKeywordScope params t | Just str:_ -> warn (getId t) 2106 $ "This only exits the subshell caused by the " ++ str ++ "." _ -> return () - else if any isFunction $ take 1 path + else case path of -- breaking at a source/function invocation is an abomination. Let's ignore it. - then err (getId t) 2104 $ "In functions, use return instead of " ++ name ++ "." - else err (getId t) 2105 $ name ++ " is only valid in loops." + h:_ | isFunction h -> err (getId t) 2104 $ "In functions, use return instead of " ++ name ++ "." + _ -> err (getId t) 2105 $ name ++ " is only valid in loops." where path = let p = getPath (parentMap params) t in NE.filter relevant p subshellType t = case leadType params t of From 7b0589988fca3bf234d2d62e475f2e821ddb42c5 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 31 Dec 2023 17:21:50 -0500 Subject: [PATCH 673/763] Implement isCondition in terms of foldr --- src/ShellCheck/Analytics.hs | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index ca679bb..3f686ee 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -346,13 +346,11 @@ dist a b hasFloatingPoint params = shellType params == Ksh -- Checks whether the current parent path is part of a condition -isCondition [] = False -isCondition [_] = False -isCondition (child:parent:rest) = - case child of - T_BatsTest {} -> True -- count anything in a @test as conditional - _ -> getId child `elem` map getId (getConditionChildren parent) || isCondition (parent:rest) +isCondition (x NE.:| xs) = foldr go (const False) xs x where + go _ _ T_BatsTest{} = True -- count anything in a @test as conditional + go parent go_rest child = + getId child `elem` map getId (getConditionChildren parent) || go_rest parent getConditionChildren t = case t of T_AndIf _ left right -> [left] @@ -886,7 +884,7 @@ checkShorthandIf params x@(T_OrIf _ (T_AndIf id _ _) (T_Pipeline _ _ t)) name <- getCommandBasename t return $ name `elem` ["echo", "exit", "return", "printf"]) isOk _ = False - inCondition = isCondition $ NE.toList $ getPath (parentMap params) x + inCondition = isCondition $ getPath (parentMap params) x checkShorthandIf _ _ = return () @@ -3185,7 +3183,7 @@ checkUncheckedCdPushdPopd params root = | name `elem` ["cd", "pushd", "popd"] && not (isSafeDir t) && not (name `elem` ["pushd", "popd"] && ("n" `elem` map snd (getAllFlags t))) - && not (isCondition $ NE.toList $ getPath (parentMap params) t) = + && not (isCondition $ getPath (parentMap params) t) = warnWithFix (getId t) 2164 ("Use '" ++ name ++ " ... || exit' or '" ++ name ++ " ... || return' in case " ++ name ++ " fails.") (fixWith [replaceEnd (getId t) params 0 " || exit"]) @@ -4002,7 +4000,7 @@ checkUselessBang params t = when (hasSetE params) $ mapM_ check (getNonReturning where check t = case t of - T_Banged id cmd | not $ isCondition (NE.toList $ getPath (parentMap params) t) -> + T_Banged id cmd | not $ isCondition (getPath (parentMap params) t) -> addComment $ makeCommentWithFix InfoC id 2251 "This ! is not on a condition and skips errexit. Use `&& exit 1` instead, or make sure $? is checked." (fixWith [replaceStart id params 1 "", replaceEnd (getId cmd) params 0 " && exit 1"]) From b7f88ec4b72bae672f1b77e4af312990f17ab86b Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 31 Dec 2023 18:09:02 -0500 Subject: [PATCH 674/763] Stop building tuples that we never look at both sides of --- src/ShellCheck/Analytics.hs | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 3f686ee..9913b09 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -4819,15 +4819,15 @@ checkExtraMaskedReturns params t = ++ "separately to avoid masking its return value (or use '|| true' " ++ "to ignore).") - isMaskDeliberate t = hasParent isOrIf t + isMaskDeliberate t = any isOrIf $ NE.init $ parents params t where - isOrIf _ (T_OrIf _ _ (T_Pipeline _ _ [T_Redirecting _ _ cmd])) + isOrIf (T_OrIf _ _ (T_Pipeline _ _ [T_Redirecting _ _ cmd])) = getCommandBasename cmd `elem` [Just "true", Just ":"] - isOrIf _ _ = False + isOrIf _ = False - isCheckedElsewhere t = hasParent isDeclaringCommand t + isCheckedElsewhere t = any isDeclaringCommand $ NE.tail $ parents params t where - isDeclaringCommand t _ = fromMaybe False $ do + isDeclaringCommand t = fromMaybe False $ do cmd <- getCommand t basename <- getCommandBasename cmd return $ @@ -4851,13 +4851,6 @@ checkExtraMaskedReturns params t = basename <- getCommandBasename t return $ basename == "time" - parentChildPairs t = go $ NE.toList $ parents params t - where - go (child:parent:rest) = (parent, child):go (parent:rest) - go _ = [] - - hasParent pred t = any (uncurry pred) (parentChildPairs t) - -- hard error on negated command that is not last prop_checkBatsTestDoesNotUseNegation1 = verify checkBatsTestDoesNotUseNegation "#!/usr/bin/env/bats\n@test \"name\" { ! true; false; }" From 9e0fdbe431f6a9725a461b214c97f885782f0314 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 31 Dec 2023 18:13:32 -0500 Subject: [PATCH 675/763] Simplify isTransparentCommand --- src/ShellCheck/Analytics.hs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 9913b09..71caa62 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -4847,9 +4847,7 @@ checkExtraMaskedReturns params t = ,"shopt" ] - isTransparentCommand t = fromMaybe False $ do - basename <- getCommandBasename t - return $ basename == "time" + isTransparentCommand t = getCommandBasename t == Just "time" -- hard error on negated command that is not last From 5a6f4840adf584da60273c6b8145c19fd7e342a4 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Mon, 1 Jan 2024 14:16:50 -0500 Subject: [PATCH 676/763] Replace a few more occurrences of !!! with pattern matching --- src/ShellCheck/ASTLib.hs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index aadff05..9b151c2 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -858,8 +858,7 @@ getBracedModifier s = headOrDefault "" $ do -- Get the variables from indices like ["x", "y"] in ${var[x+y+1]} prop_getIndexReferences1 = getIndexReferences "var[x+y+1]" == ["x", "y"] getIndexReferences s = fromMaybe [] $ do - match <- matchRegex re s - index <- match !!! 0 + index:_ <- matchRegex re s return $ matchAllStrings variableNameRegex index where re = mkRegex "(\\[.*\\])" @@ -870,8 +869,7 @@ prop_getOffsetReferences3 = getOffsetReferences "[foo]:bar" == ["bar"] prop_getOffsetReferences4 = getOffsetReferences "[foo]:bar:baz" == ["bar", "baz"] getOffsetReferences mods = fromMaybe [] $ do -- if mods start with [, then drop until ] - match <- matchRegex re mods - offsets <- match !!! 1 + _:offsets:_ <- matchRegex re mods return $ matchAllStrings variableNameRegex offsets where re = mkRegex "^(\\[.+\\])? *:([^-=?+].*)" From 025cc5266ec1632d5f524644f0366237f604e7e8 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Mon, 1 Jan 2024 16:00:19 -0500 Subject: [PATCH 677/763] Simplify isUnquotedFlag --- src/ShellCheck/ASTLib.hs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 9b151c2..88089ea 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -158,9 +158,10 @@ isFlag token = _ -> False -- Is this token a flag where the - is unquoted? -isUnquotedFlag token = fromMaybe False $ do - str <- getLeadingUnquotedString token - return $ "-" `isPrefixOf` str +isUnquotedFlag token = + case getLeadingUnquotedString token of + Just ('-':_) -> True + _ -> False -- getGnuOpts "erd:u:" will parse a list of arguments tokens like `read` -- -re -d : -u 3 bar From 67abfe159e41e33a7ad17d9b5e130756cc447510 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Mon, 1 Jan 2024 19:04:26 -0500 Subject: [PATCH 678/763] Remove most of the partial head and tail functions from src/ShellCheck/CFG.hs --- src/ShellCheck/CFG.hs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/ShellCheck/CFG.hs b/src/ShellCheck/CFG.hs index 5d018c8..e1d3259 100644 --- a/src/ShellCheck/CFG.hs +++ b/src/ShellCheck/CFG.hs @@ -615,15 +615,15 @@ build t = do T_CaseExpression id t [] -> build t - T_CaseExpression id t list -> do + T_CaseExpression id t list@(hd:tl) -> do start <- newStructuralNode token <- build t - branches <- mapM buildBranch list + branches <- mapM buildBranch (hd NE.:| tl) end <- newStructuralNode - let neighbors = zip branches $ tail branches - let (_, firstCond, _) = head branches - let (_, lastCond, lastBody) = last branches + let neighbors = zip (NE.toList branches) $ NE.tail branches + let (_, firstCond, _) = NE.head branches + let (_, lastCond, lastBody) = NE.last branches linkRange start token linkRange token firstCond From ba86c6363c30a5dbefd0b8b9a7c5f4ab0478dc91 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Tue, 2 Jan 2024 14:46:07 -0500 Subject: [PATCH 679/763] Use maybe instead of fromMaybe and fmap --- src/ShellCheck/Analytics.hs | 2 +- src/ShellCheck/CFGAnalysis.hs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 71caa62..3af0455 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -5020,7 +5020,7 @@ checkPlusEqualsNumber params t = let unquotedLiteral = getUnquotedLiteral word isEmpty = unquotedLiteral == Just "" - isUnquotedNumber = not isEmpty && fromMaybe False (all isDigit <$> unquotedLiteral) + isUnquotedNumber = not isEmpty && maybe False (all isDigit) unquotedLiteral isNumericalVariableName = fromMaybe False $ do str <- unquotedLiteral CF.variableMayBeAssignedInteger state str diff --git a/src/ShellCheck/CFGAnalysis.hs b/src/ShellCheck/CFGAnalysis.hs index 16afa68..0b99c9f 100644 --- a/src/ShellCheck/CFGAnalysis.hs +++ b/src/ShellCheck/CFGAnalysis.hs @@ -829,7 +829,7 @@ lookupStack' functionOnly get dep def ctx key = do f (s:rest) = do -- Go up the stack until we find the value, and add -- a dependency on each state (including where it was found) - res <- fromMaybe (f rest) (return <$> get (stackState s) key) + res <- maybe (f rest) return (get (stackState s) key) modifySTRef (dependencies s) $ S.insert $ dep key res return res From 1bce426fcfd6ff176800bd70d7a298cf261d8512 Mon Sep 17 00:00:00 2001 From: Georg Pfuetzenreuter Date: Sun, 21 Jan 2024 02:16:35 +0100 Subject: [PATCH 680/763] Implement rcfile option This introduces the "--rcfile" argument which allows a specific shellcheckrc file to be passed. If specified and the given file exists, the default locations will not be searched and the specified file will be used. Signed-off-by: Georg Pfuetzenreuter --- shellcheck.1.md | 5 +++++ shellcheck.hs | 39 +++++++++++++++++++++++++++------------ 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/shellcheck.1.md b/shellcheck.1.md index 89f6d50..42a0429 100644 --- a/shellcheck.1.md +++ b/shellcheck.1.md @@ -71,6 +71,11 @@ not warn at all, as `ksh` supports decimals in arithmetic contexts. : Don't try to look for .shellcheckrc configuration files. +--rcfile\ RCFILE + +: Prefer the specified configuration file over searching for one + in the default locations. + **-o**\ *NAME1*[,*NAME2*...],\ **--enable=***NAME1*[,*NAME2*...] : Enable optional checks. The special name *all* enables all of them. diff --git a/shellcheck.hs b/shellcheck.hs index 00b699b..42554f3 100644 --- a/shellcheck.hs +++ b/shellcheck.hs @@ -76,7 +76,8 @@ data Options = Options { externalSources :: Bool, sourcePaths :: [FilePath], formatterOptions :: FormatterOptions, - minSeverity :: Severity + minSeverity :: Severity, + rcfile :: FilePath } defaultOptions = Options { @@ -86,7 +87,8 @@ defaultOptions = Options { formatterOptions = newFormatterOptions { foColorOption = ColorAuto }, - minSeverity = StyleC + minSeverity = StyleC, + rcfile = [] } usageHeader = "Usage: shellcheck [OPTIONS...] FILES..." @@ -107,6 +109,9 @@ options = [ (NoArg $ Flag "list-optional" "true") "List checks disabled by default", Option "" ["norc"] (NoArg $ Flag "norc" "true") "Don't look for .shellcheckrc files", + Option "" ["rcfile"] + (ReqArg (Flag "rcfile") "RCFILE") + "Prefer the specified configuration file over searching for one", Option "o" ["enable"] (ReqArg (Flag "enable") "check1,check2..") "List of optional checks to enable (or 'all')", @@ -367,6 +372,11 @@ parseOption flag options = } } + Flag "rcfile" str -> do + return options { + rcfile = str + } + Flag "enable" value -> let cs = checkSpec options in return options { checkSpec = cs { @@ -441,18 +451,23 @@ ioInterface options files = do fallback :: FilePath -> IOException -> IO FilePath fallback path _ = return path + -- Returns the name and contents of .shellcheckrc for the given file getConfig cache filename = do - path <- normalize filename - let dir = takeDirectory path - (previousPath, result) <- readIORef cache - if dir == previousPath - then return result - else do - paths <- getConfigPaths dir - result <- findConfig paths - writeIORef cache (dir, result) - return result + contents <- readConfig (rcfile options) + if isNothing contents + then do + path <- normalize filename + let dir = takeDirectory path + (previousPath, result) <- readIORef cache + if dir == previousPath + then return result + else do + paths <- getConfigPaths dir + result <- findConfig paths + writeIORef cache (dir, result) + return result + else return contents findConfig paths = case paths of From de95624d310622d234149e5b1bc96da2a40ff317 Mon Sep 17 00:00:00 2001 From: Grische <2787581+grische@users.noreply.github.com> Date: Fri, 2 Feb 2024 12:28:09 +0100 Subject: [PATCH 681/763] Remove deprecated "install --enable-tests" command --- README.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/README.md b/README.md index ca9b847..3692319 100644 --- a/README.md +++ b/README.md @@ -309,10 +309,6 @@ Verify that `cabal` is installed and update its dependency list with $ cabal install -Or if you intend to run the tests: - - $ cabal install --enable-tests - This will compile ShellCheck and install it to your `~/.cabal/bin` directory. Add this directory to your `PATH` (for bash, add this to your `~/.bashrc`): @@ -558,4 +554,3 @@ Happy ShellChecking! * The wiki has [long form descriptions](https://github.com/koalaman/shellcheck/wiki/Checks) for each warning, e.g. [SC2221](https://github.com/koalaman/shellcheck/wiki/SC2221). * ShellCheck does not attempt to enforce any kind of formatting or indenting style, so also check out [shfmt](https://github.com/mvdan/sh)! - From 6a44a19f17c4a0590693587af3b5209d7b1b59fe Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 3 Feb 2024 13:34:49 -0800 Subject: [PATCH 682/763] Only read --rcfile once, and skip search if unavailable --- shellcheck.hs | 48 +++++++++++++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/shellcheck.hs b/shellcheck.hs index 42554f3..e933d6c 100644 --- a/shellcheck.hs +++ b/shellcheck.hs @@ -77,7 +77,7 @@ data Options = Options { sourcePaths :: [FilePath], formatterOptions :: FormatterOptions, minSeverity :: Severity, - rcfile :: FilePath + rcfile :: Maybe FilePath } defaultOptions = Options { @@ -88,7 +88,7 @@ defaultOptions = Options { foColorOption = ColorAuto }, minSeverity = StyleC, - rcfile = [] + rcfile = Nothing } usageHeader = "Usage: shellcheck [OPTIONS...] FILES..." @@ -374,7 +374,7 @@ parseOption flag options = Flag "rcfile" str -> do return options { - rcfile = str + rcfile = Just str } Flag "enable" value -> @@ -453,21 +453,31 @@ ioInterface options files = do -- Returns the name and contents of .shellcheckrc for the given file - getConfig cache filename = do - contents <- readConfig (rcfile options) - if isNothing contents - then do - path <- normalize filename - let dir = takeDirectory path - (previousPath, result) <- readIORef cache - if dir == previousPath - then return result - else do - paths <- getConfigPaths dir - result <- findConfig paths - writeIORef cache (dir, result) - return result - else return contents + getConfig cache filename = + case rcfile options of + Just file -> do + -- We have a specified rcfile. Ignore normal rcfile resolution. + (path, result) <- readIORef cache + if path == "/" + then return result + else do + result <- readConfig file + when (isNothing result) $ + hPutStrLn stderr $ "Warning: unable to read --rcfile " ++ file + writeIORef cache ("/", result) + return result + + Nothing -> do + path <- normalize filename + let dir = takeDirectory path + (previousPath, result) <- readIORef cache + if dir == previousPath + then return result + else do + paths <- getConfigPaths dir + result <- findConfig paths + writeIORef cache (dir, result) + return result findConfig paths = case paths of @@ -505,7 +515,7 @@ ioInterface options files = do where handler :: FilePath -> IOException -> IO (String, Bool) handler file err = do - putStrLn $ file ++ ": " ++ show err + hPutStrLn stderr $ file ++ ": " ++ show err return ("", True) andM a b arg = do From d80fdfa9e8e738827a88505b26d3e596c0f0e875 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 3 Feb 2024 15:45:23 -0800 Subject: [PATCH 683/763] Add extended-analysis directive to toggle DFA --- CHANGELOG.md | 5 +++- shellcheck.1.md | 13 +++++++++ shellcheck.hs | 18 +++++++++++++ src/ShellCheck/AST.hs | 1 + src/ShellCheck/ASTLib.hs | 6 +++++ src/ShellCheck/Analytics.hs | 21 +++++++++------ src/ShellCheck/AnalyzerLib.hs | 8 ++++-- src/ShellCheck/Checker.hs | 40 ++++++++++++++++++++++++++++ src/ShellCheck/Checks/Commands.hs | 18 +++++++------ src/ShellCheck/Checks/ControlFlow.hs | 2 +- src/ShellCheck/Interface.hs | 8 ++++-- src/ShellCheck/Parser.hs | 10 +++++++ 12 files changed, 128 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 897aa27..dc7f6ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ ## Git ### Added +- Added support for busybox sh +- Added flag --rcfile to specify an rc file by name. +- Added `extended-analysis=true` directive to enable/disable dataflow analysis + (with a corresponding --extended-analysis flag). - SC2324: Warn when x+=1 appends instead of increments - SC2325: Warn about multiple `!`s in dash/sh. - SC2326: Warn about `foo | ! bar` in bash/dash/sh. @@ -9,7 +13,6 @@ - SC3015: Warn bashism `test _ =~ _` like in [ ] - SC3016: Warn bashism `test -v _` like in [ ] - SC3017: Warn bashism `test -a _` like in [ ] -- Added support for busybox sh ### Fixed - source statements with here docs now work correctly diff --git a/shellcheck.1.md b/shellcheck.1.md index 42a0429..b2bef3c 100644 --- a/shellcheck.1.md +++ b/shellcheck.1.md @@ -56,6 +56,13 @@ not warn at all, as `ksh` supports decimals in arithmetic contexts. options are cumulative, but all the codes can be specified at once, comma-separated as a single argument. +**--extended-analysis=true/false** + +: Enable/disable Dataflow Analysis to identify more issues (default true). If + ShellCheck uses too much CPU/RAM when checking scripts with several + thousand lines of code, extended analysis can be disabled with this flag + or a directive. This flag overrides directives and rc files. + **-f** *FORMAT*, **--format=***FORMAT* : Specify the output format of shellcheck, which prints its results in the @@ -249,6 +256,12 @@ Valid keys are: : Enable an optional check by name, as listed with **--list-optional**. Only file-wide `enable` directives are considered. +**extended-analysis** +: Set to true/false to enable/disable dataflow analysis. Specifying + `# shellcheck extended-analysis=false` in particularly large (2000+ line) + auto-generated scripts will reduce ShellCheck's resource usage at the + expense of certain checks. Extended analysis is enabled by default. + **external-sources** : Set to `true` in `.shellcheckrc` to always allow ShellCheck to open arbitrary files from 'source' statements (the way most tools do). diff --git a/shellcheck.hs b/shellcheck.hs index e933d6c..def3654 100644 --- a/shellcheck.hs +++ b/shellcheck.hs @@ -102,6 +102,8 @@ options = [ (ReqArg (Flag "include") "CODE1,CODE2..") "Consider only given types of warnings", Option "e" ["exclude"] (ReqArg (Flag "exclude") "CODE1,CODE2..") "Exclude types of warnings", + Option "" ["extended-analysis"] + (ReqArg (Flag "extended-analysis") "bool") "Perform dataflow analysis (default true)", Option "f" ["format"] (ReqArg (Flag "format") "FORMAT") $ "Output format (" ++ formatList ++ ")", @@ -384,6 +386,14 @@ parseOption flag options = } } + Flag "extended-analysis" str -> do + value <- parseBool str + return options { + checkSpec = (checkSpec options) { + csExtendedAnalysis = Just value + } + } + -- This flag is handled specially in 'process' Flag "format" _ -> return options @@ -401,6 +411,14 @@ parseOption flag options = throwError SyntaxFailure return (Prelude.read num :: Integer) + parseBool str = do + case str of + "true" -> return True + "false" -> return False + _ -> do + printErr $ "Invalid boolean, expected true/false: " ++ str + throwError SyntaxFailure + ioInterface :: Options -> [FilePath] -> IO (SystemInterface IO) ioInterface options files = do inputs <- mapM normalize files diff --git a/src/ShellCheck/AST.hs b/src/ShellCheck/AST.hs index 5c20416..ca05c98 100644 --- a/src/ShellCheck/AST.hs +++ b/src/ShellCheck/AST.hs @@ -152,6 +152,7 @@ data Annotation = | ShellOverride String | SourcePath String | ExternalSources Bool + | ExtendedAnalysis Bool deriving (Show, Eq) data ConditionType = DoubleBracket | SingleBracket deriving (Show, Eq) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 88089ea..6b26b22 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -910,5 +910,11 @@ getEnableDirectives root = T_Annotation _ list _ -> [s | EnableComment s <- list] _ -> [] +getExtendedAnalysisDirective :: Token -> Maybe Bool +getExtendedAnalysisDirective root = + case root of + T_Annotation _ list _ -> listToMaybe $ [s | ExtendedAnalysis s <- list] + _ -> Nothing + return [] runTests = $quickCheckAll diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 3af0455..f885842 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1262,7 +1262,8 @@ checkNumberComparisons params (TC_Binary id typ op lhs rhs) = do str = concat $ oversimplify c var = getBracedReference str in fromMaybe False $ do - state <- CF.getIncomingState (cfgAnalysis params) id + cfga <- cfgAnalysis params + state <- CF.getIncomingState cfga id value <- Map.lookup var $ CF.variablesInScope state return $ CF.numericalStatus (CF.variableValue value) >= CF.NumericalStatusMaybe _ -> @@ -2143,7 +2144,8 @@ checkSpacefulnessCfg' dirtyPass params token@(T_DollarBraced id _ list) = && not (usedAsCommandName parents token) isClean = fromMaybe False $ do - state <- CF.getIncomingState (cfgAnalysis params) id + cfga <- cfgAnalysis params + state <- CF.getIncomingState cfga id value <- Map.lookup name $ CF.variablesInScope state return $ isCleanState value @@ -4896,7 +4898,8 @@ prop_checkCommandIsUnreachable3 = verifyNot checkCommandIsUnreachable "foo; bar checkCommandIsUnreachable params t = case t of T_Pipeline {} -> sequence_ $ do - state <- CF.getIncomingState (cfgAnalysis params) id + cfga <- cfgAnalysis params + state <- CF.getIncomingState cfga id guard . not $ CF.stateIsReachable state guard . not $ isSourced params t return $ info id 2317 "Command appears to be unreachable. Check usage (or ignore if invoked indirectly)." @@ -4918,14 +4921,15 @@ checkOverwrittenExitCode params t = _ -> return () where check id = sequence_ $ do - state <- CF.getIncomingState (cfgAnalysis params) id + cfga <- cfgAnalysis params + state <- CF.getIncomingState cfga id let exitCodeIds = CF.exitCodes state guard . not $ S.null exitCodeIds let idToToken = idMap params exitCodeTokens <- traverse (\k -> Map.lookup k idToToken) $ S.toList exitCodeIds return $ do - when (all isCondition exitCodeTokens && not (usedUnconditionally t exitCodeIds)) $ + when (all isCondition exitCodeTokens && not (usedUnconditionally cfga t exitCodeIds)) $ warn id 2319 "This $? refers to a condition, not a command. Assign to a variable to avoid it being overwritten." when (all isPrinting exitCodeTokens) $ warn id 2320 "This $? refers to echo/printf, not a previous command. Assign to variable to avoid it being overwritten." @@ -4938,8 +4942,8 @@ checkOverwrittenExitCode params t = -- If we don't do anything based on the condition, assume we wanted the condition itself -- This helps differentiate `x; [ $? -gt 0 ] && exit $?` vs `[ cond ]; exit $?` - usedUnconditionally t testIds = - all (\c -> CF.doesPostDominate (cfgAnalysis params) (getId t) c) testIds + usedUnconditionally cfga t testIds = + all (\c -> CF.doesPostDominate cfga (getId t) c) testIds isPrinting t = case getCommandBasename t of @@ -5009,7 +5013,8 @@ prop_checkPlusEqualsNumber9 = verifyNot checkPlusEqualsNumber "declare -ia var; checkPlusEqualsNumber params t = case t of T_Assignment id Append var _ word -> sequence_ $ do - state <- CF.getIncomingState (cfgAnalysis params) id + cfga <- cfgAnalysis params + state <- CF.getIncomingState cfga id guard $ isNumber state word guard . not $ fromMaybe False $ CF.variableMayBeDeclaredInteger state var return $ warn id 2324 "var+=1 will append, not increment. Use (( var += 1 )), declare -i var, or quote number to silence." diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 944b12d..d265ace 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -104,7 +104,7 @@ data Parameters = Parameters { -- map from token id to start and end position tokenPositions :: Map.Map Id (Position, Position), -- Result from Control Flow Graph analysis (including data flow analysis) - cfgAnalysis :: CF.CFGAnalysis + cfgAnalysis :: Maybe CF.CFGAnalysis } deriving (Show) -- TODO: Cache results of common AST ops here @@ -197,8 +197,10 @@ makeCommentWithFix severity id code str fix = } in force withFix +-- makeParameters :: CheckSpec -> Parameters makeParameters spec = params where + extendedAnalysis = fromMaybe True $ msum [asExtendedAnalysis spec, getExtendedAnalysisDirective root] params = Parameters { rootNode = root, shellType = fromMaybe (determineShell (asFallbackShell spec) root) $ asShellType spec, @@ -229,7 +231,9 @@ makeParameters spec = params parentMap = getParentTree root, variableFlow = getVariableFlow params root, tokenPositions = asTokenPositions spec, - cfgAnalysis = CF.analyzeControlFlow cfParams root + cfgAnalysis = do + guard extendedAnalysis + return $ CF.analyzeControlFlow cfParams root } cfParams = CF.CFGParameters { CF.cfLastpipe = hasLastpipe params, diff --git a/src/ShellCheck/Checker.hs b/src/ShellCheck/Checker.hs index 6c9166f..0cfc3ab 100644 --- a/src/ShellCheck/Checker.hs +++ b/src/ShellCheck/Checker.hs @@ -25,6 +25,7 @@ import ShellCheck.ASTLib import ShellCheck.Interface import ShellCheck.Parser +import Debug.Trace -- DO NOT SUBMIT import Data.Either import Data.Functor import Data.List @@ -86,6 +87,7 @@ checkScript sys spec = do asCheckSourced = csCheckSourced spec, asExecutionMode = Executed, asTokenPositions = tokenPositions, + asExtendedAnalysis = csExtendedAnalysis spec, asOptionalChecks = getEnableDirectives root ++ csOptionalChecks spec } where as = newAnalysisSpec root let analysisMessages = @@ -520,5 +522,43 @@ prop_hereDocsWillHaveParsedIndices = null result where result = check "#!/bin/bash\nmy_array=(a b)\ncat <> ./test\n $(( 1 + my_array[1] ))\nEOF" +prop_rcCanSuppressDfa = null result + where + result = checkWithRc "extended-analysis=false" emptyCheckSpec { + csScript = "#!/bin/sh\nexit; foo;" + } + +prop_fileCanSuppressDfa = null $ traceShowId result + where + result = checkWithRc "" emptyCheckSpec { + csScript = "#!/bin/sh\n# shellcheck extended-analysis=false\nexit; foo;" + } + +prop_fileWinsWhenSuppressingDfa1 = null result + where + result = checkWithRc "extended-analysis=true" emptyCheckSpec { + csScript = "#!/bin/sh\n# shellcheck extended-analysis=false\nexit; foo;" + } + +prop_fileWinsWhenSuppressingDfa2 = result == [2317] + where + result = checkWithRc "extended-analysis=false" emptyCheckSpec { + csScript = "#!/bin/sh\n# shellcheck extended-analysis=true\nexit; foo;" + } + +prop_flagWinsWhenSuppressingDfa1 = result == [2317] + where + result = checkWithRc "extended-analysis=false" emptyCheckSpec { + csScript = "#!/bin/sh\n# shellcheck extended-analysis=false\nexit; foo;", + csExtendedAnalysis = Just True + } + +prop_flagWinsWhenSuppressingDfa2 = null result + where + result = checkWithRc "extended-analysis=true" emptyCheckSpec { + csScript = "#!/bin/sh\n# shellcheck extended-analysis=true\nexit; foo;", + csExtendedAnalysis = Just False + } + return [] runTests = $quickCheckAll diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 97c9088..c10016e 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -1430,26 +1430,28 @@ prop_checkBackreferencingDeclaration6 = verify (checkBackreferencingDeclaration prop_checkBackreferencingDeclaration7 = verify (checkBackreferencingDeclaration "declare") "declare x=var $k=$x" checkBackreferencingDeclaration cmd = CommandCheck (Exactly cmd) check where - check t = foldM_ perArg M.empty $ arguments t + check t = do + cfga <- asks cfgAnalysis + when (isJust cfga) $ + foldM_ (perArg $ fromJust cfga) M.empty $ arguments t - perArg leftArgs t = + perArg cfga leftArgs t = case t of T_Assignment id _ name idx t -> do - warnIfBackreferencing leftArgs $ t:idx + warnIfBackreferencing cfga leftArgs $ t:idx return $ M.insert name id leftArgs t -> do - warnIfBackreferencing leftArgs [t] + warnIfBackreferencing cfga leftArgs [t] return leftArgs - warnIfBackreferencing backrefs l = do - references <- findReferences l + warnIfBackreferencing cfga backrefs l = do + references <- findReferences cfga l let reused = M.intersection backrefs references mapM msg $ M.toList reused msg (name, id) = warn id 2318 $ "This assignment is used again in this '" ++ cmd ++ "', but won't have taken effect. Use two '" ++ cmd ++ "'s." - findReferences list = do - cfga <- asks cfgAnalysis + findReferences cfga list = do let graph = CF.graph cfga let nodesMap = CF.tokenToNodes cfga let nodes = S.unions $ map (\id -> M.findWithDefault S.empty id nodesMap) $ map getId $ list diff --git a/src/ShellCheck/Checks/ControlFlow.hs b/src/ShellCheck/Checks/ControlFlow.hs index 9b7635e..d23fa15 100644 --- a/src/ShellCheck/Checks/ControlFlow.hs +++ b/src/ShellCheck/Checks/ControlFlow.hs @@ -78,7 +78,7 @@ controlFlowEffectChecks = [ runNodeChecks :: [ControlFlowNodeCheck] -> ControlFlowCheck runNodeChecks perNode = do cfg <- asks cfgAnalysis - runOnAll cfg + sequence_ $ runOnAll <$> cfg where getData datas n@(node, label) = do (pre, post) <- M.lookup node datas diff --git a/src/ShellCheck/Interface.hs b/src/ShellCheck/Interface.hs index c574cee..04e3c5a 100644 --- a/src/ShellCheck/Interface.hs +++ b/src/ShellCheck/Interface.hs @@ -21,11 +21,11 @@ module ShellCheck.Interface ( SystemInterface(..) - , CheckSpec(csFilename, csScript, csCheckSourced, csIncludedWarnings, csExcludedWarnings, csShellTypeOverride, csMinSeverity, csIgnoreRC, csOptionalChecks) + , CheckSpec(csFilename, csScript, csCheckSourced, csIncludedWarnings, csExcludedWarnings, csShellTypeOverride, csMinSeverity, csIgnoreRC, csExtendedAnalysis, csOptionalChecks) , CheckResult(crFilename, crComments) , ParseSpec(psFilename, psScript, psCheckSourced, psIgnoreRC, psShellTypeOverride) , ParseResult(prComments, prTokenPositions, prRoot) - , AnalysisSpec(asScript, asShellType, asFallbackShell, asExecutionMode, asCheckSourced, asTokenPositions, asOptionalChecks) + , AnalysisSpec(asScript, asShellType, asFallbackShell, asExecutionMode, asCheckSourced, asTokenPositions, asExtendedAnalysis, asOptionalChecks) , AnalysisResult(arComments) , FormatterOptions(foColorOption, foWikiLinkCount) , Shell(Ksh, Sh, Bash, Dash, BusyboxSh) @@ -100,6 +100,7 @@ data CheckSpec = CheckSpec { csIncludedWarnings :: Maybe [Integer], csShellTypeOverride :: Maybe Shell, csMinSeverity :: Severity, + csExtendedAnalysis :: Maybe Bool, csOptionalChecks :: [String] } deriving (Show, Eq) @@ -124,6 +125,7 @@ emptyCheckSpec = CheckSpec { csIncludedWarnings = Nothing, csShellTypeOverride = Nothing, csMinSeverity = StyleC, + csExtendedAnalysis = Nothing, csOptionalChecks = [] } @@ -174,6 +176,7 @@ data AnalysisSpec = AnalysisSpec { asExecutionMode :: ExecutionMode, asCheckSourced :: Bool, asOptionalChecks :: [String], + asExtendedAnalysis :: Maybe Bool, asTokenPositions :: Map.Map Id (Position, Position) } @@ -184,6 +187,7 @@ newAnalysisSpec token = AnalysisSpec { asExecutionMode = Executed, asCheckSourced = False, asOptionalChecks = [], + asExtendedAnalysis = Nothing, asTokenPositions = Map.empty } diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 130d956..9cc5e02 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -1058,6 +1058,16 @@ readAnnotationWithoutPrefix sandboxed = do "This shell type is unknown. Use e.g. sh or bash." return [ShellOverride shell] + "extended-analysis" -> do + pos <- getPosition + value <- plainOrQuoted $ many1 letter + case value of + "true" -> return [ExtendedAnalysis True] + "false" -> return [ExtendedAnalysis False] + _ -> do + parseNoteAt pos ErrorC 1146 "Unknown extended-analysis value. Expected true/false." + return [] + "external-sources" -> do pos <- getPosition value <- plainOrQuoted $ many1 letter From 8c4c112c2504b630b5e58a87c8f42bbe9c955946 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 19 Feb 2024 09:28:04 -0800 Subject: [PATCH 684/763] Initial version of an ARM64 macOS build --- .github/workflows/build.yml | 2 +- build/darwin.aarch64/Dockerfile | 40 +++++++++++++++++++++++++++++++++ build/darwin.aarch64/build | 15 +++++++++++++ build/darwin.aarch64/tag | 1 + 4 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 build/darwin.aarch64/Dockerfile create mode 100755 build/darwin.aarch64/build create mode 100644 build/darwin.aarch64/tag diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3e6fb27..c49b25a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -47,7 +47,7 @@ jobs: needs: package_source strategy: matrix: - build: [linux.x86_64, linux.aarch64, linux.armv6hf, darwin.x86_64, windows.x86_64] + build: [linux.x86_64, linux.aarch64, linux.armv6hf, darwin.x86_64, darwin.aarch64, windows.x86_64] runs-on: ubuntu-latest steps: - name: Checkout repository diff --git a/build/darwin.aarch64/Dockerfile b/build/darwin.aarch64/Dockerfile new file mode 100644 index 0000000..7839728 --- /dev/null +++ b/build/darwin.aarch64/Dockerfile @@ -0,0 +1,40 @@ +FROM ghcr.io/shepherdjerred/macos-cross-compiler:latest + +ENV TARGET aarch64-apple-darwin22 +ENV TARGETNAME darwin.aarch64 + +# Build dependencies +USER root +ENV DEBIAN_FRONTEND noninteractive +ENV LC_ALL C.utf8 + +# Install basic deps +RUN apt-get update && apt-get install -y automake autoconf build-essential curl xz-utils qemu-user-static + +# Install a more suitable host compiler +WORKDIR /host-ghc +RUN curl -L "https://downloads.haskell.org/~cabal/cabal-install-3.9.0.0/cabal-install-3.9-x86_64-linux-alpine.tar.xz" | tar xJv -C /usr/local/bin +RUN curl -L 'https://downloads.haskell.org/~ghc/8.10.7/ghc-8.10.7-x86_64-deb10-linux.tar.xz' | tar xJ --strip-components=1 +RUN ./configure && make install + +# Build GHC. We have to use an old version because cross-compilation across OS has since broken. +WORKDIR /ghc +RUN curl -L "https://downloads.haskell.org/~ghc/8.10.7/ghc-8.10.7-src.tar.xz" | tar xJ --strip-components=1 +RUN apt-get install -y llvm-12 +RUN ./boot && ./configure --host x86_64-linux-gnu --build x86_64-linux-gnu --target "$TARGET" +RUN cp mk/flavours/quick-cross.mk mk/build.mk && make -j "$(nproc)" +RUN make install + +# Due to an apparent cabal bug, we specify our options directly to cabal +# It won't reuse caches if ghc-options are specified in ~/.cabal/config +ENV CABALOPTS "--ghc-options;-optc-Os -optc-fPIC;--with-ghc=$TARGET-ghc;--with-hc-pkg=$TARGET-ghc-pkg;--constraint=hashable==1.3.5.0" + +# Prebuild the dependencies +RUN cabal update +RUN IFS=';' && cabal install --dependencies-only $CABALOPTS ShellCheck + +# Copy the build script +COPY build /usr/bin + +WORKDIR /scratch +ENTRYPOINT ["/usr/bin/build"] diff --git a/build/darwin.aarch64/build b/build/darwin.aarch64/build new file mode 100755 index 0000000..c15717a --- /dev/null +++ b/build/darwin.aarch64/build @@ -0,0 +1,15 @@ +#!/bin/sh +set -xe +{ + tar xzv --strip-components=1 + chmod +x striptests && ./striptests + mkdir "$TARGETNAME" + cabal update + ( IFS=';'; cabal build $CABALOPTS ) + find . -name shellcheck -type f -exec mv {} "$TARGETNAME/" \; + ls -l "$TARGETNAME" + "$TARGET-strip" "$TARGETNAME/shellcheck" + ls -l "$TARGETNAME" + file "$TARGETNAME/shellcheck" | grep "Mach-O 64-bit arm64 executable" +} >&2 +tar czv "$TARGETNAME" diff --git a/build/darwin.aarch64/tag b/build/darwin.aarch64/tag new file mode 100644 index 0000000..ae93ef3 --- /dev/null +++ b/build/darwin.aarch64/tag @@ -0,0 +1 @@ +koalaman/scbuilder-darwin-aarch64 From 55be4543f225824e1f6534973ad51c7833343e4e Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 19 Feb 2024 11:40:30 -0800 Subject: [PATCH 685/763] Avoid stripping darwin.aarch64 binaries to keep code signature --- build/darwin.aarch64/build | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build/darwin.aarch64/build b/build/darwin.aarch64/build index c15717a..4235d3a 100755 --- a/build/darwin.aarch64/build +++ b/build/darwin.aarch64/build @@ -8,7 +8,9 @@ set -xe ( IFS=';'; cabal build $CABALOPTS ) find . -name shellcheck -type f -exec mv {} "$TARGETNAME/" \; ls -l "$TARGETNAME" - "$TARGET-strip" "$TARGETNAME/shellcheck" + # Stripping invalidates the code signature and the build image does + # not appear to have anything similar to the 'codesign' tool. + # "$TARGET-strip" "$TARGETNAME/shellcheck" ls -l "$TARGETNAME" file "$TARGETNAME/shellcheck" | grep "Mach-O 64-bit arm64 executable" } >&2 From ad3c3146f0e10fcd331a1c8bba29e710f4417c99 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 3 Mar 2024 12:34:29 -0800 Subject: [PATCH 686/763] Fix snap build --- .snapsquid.conf | 14 -------------- snap/snapcraft.yaml | 18 +++++++++--------- 2 files changed, 9 insertions(+), 23 deletions(-) delete mode 100644 .snapsquid.conf diff --git a/.snapsquid.conf b/.snapsquid.conf deleted file mode 100644 index 205c1a6..0000000 --- a/.snapsquid.conf +++ /dev/null @@ -1,14 +0,0 @@ -# In 2015, cabal-install had a http bug triggered when proxies didn't keep -# the connection open. This version made it into Ubuntu Xenial as used by -# Snapcraft. In June 2018, Snapcraft's proxy started triggering this bug. -# -# https://bugs.launchpad.net/launchpad-buildd/+bug/1797809 -# -# Workaround: add more proxy - -visible_hostname localhost -http_port 8888 -cache_peer 10.10.10.1 parent 8222 0 no-query default -cache_peer_domain localhost !.internal -http_access allow all - diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index e14b854..f294c4e 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -23,7 +23,7 @@ description: | # snap connect shellcheck:removable-media version: git -base: core18 +base: core20 grade: stable confinement: strict @@ -40,16 +40,16 @@ parts: source: . build-packages: - cabal-install - - squid + stage-packages: + - libatomic1 override-build: | - # See comments in .snapsquid.conf - [ "$http_proxy" ] && { - squid3 -f .snapsquid.conf - export http_proxy="http://localhost:8888" - sleep 3 - } + # Give ourselves enough memory to build + dd if=/dev/zero of=/tmp/swap bs=1M count=2000 + mkswap /tmp/swap + swapon /tmp/swap + cabal sandbox init - cabal update || cat /var/log/squid/* + cabal update cabal install -j install -d $SNAPCRAFT_PART_INSTALL/usr/bin From 8bc7345aa7ec55f79a433388f443dddf4e89e270 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 3 Mar 2024 16:11:44 -0800 Subject: [PATCH 687/763] Remove outdated distros from testing --- test/distrotest | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/distrotest b/test/distrotest index 4ad66f8..ef467b8 100755 --- a/test/distrotest +++ b/test/distrotest @@ -76,8 +76,6 @@ archlinux:latest pacman -S -y --noconfirm cabal-install ghc-static base-dev # Ubuntu LTS ubuntu:22.04 apt-get update && apt-get install -y cabal-install ubuntu:20.04 apt-get update && apt-get install -y cabal-install -ubuntu:18.04 apt-get update && apt-get install -y cabal-install -ubuntu:16.04 apt-get update && apt-get install -y cabal-install # Stack on Ubuntu LTS ubuntu:22.04 set -e; apt-get update && apt-get install -y curl && curl -sSL https://get.haskellstack.org/ | sh -s - -f && cd /mnt && exec test/stacktest From a7e65dca8d7b0d19db9808e9ae17e2aa86ddbba4 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 4 Mar 2024 09:19:51 -0800 Subject: [PATCH 688/763] Update some copyright years --- shellcheck.1.md | 2 +- src/ShellCheck/Analytics.hs | 2 +- src/ShellCheck/Interface.hs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/shellcheck.1.md b/shellcheck.1.md index b2bef3c..b873e45 100644 --- a/shellcheck.1.md +++ b/shellcheck.1.md @@ -397,7 +397,7 @@ long list of wonderful contributors. # COPYRIGHT -Copyright 2012-2022, Vidar Holen and contributors. +Copyright 2012-2024, Vidar Holen and contributors. Licensed under the GNU General Public License version 3 or later, see https://gnu.org/licenses/gpl.html diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index f885842..1cc8bf8 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1,5 +1,5 @@ {- - Copyright 2012-2022 Vidar Holen + Copyright 2012-2024 Vidar Holen This file is part of ShellCheck. https://www.shellcheck.net diff --git a/src/ShellCheck/Interface.hs b/src/ShellCheck/Interface.hs index 04e3c5a..16a7e36 100644 --- a/src/ShellCheck/Interface.hs +++ b/src/ShellCheck/Interface.hs @@ -1,5 +1,5 @@ {- - Copyright 2012-2019 Vidar Holen + Copyright 2012-2024 Vidar Holen This file is part of ShellCheck. https://www.shellcheck.net From 37dfb67768db726092fde482d338943d678e6988 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 7 Mar 2024 17:53:15 -0800 Subject: [PATCH 689/763] Stable version v0.10.0 This release is dedicated to LLMs, for finally fulfilling the promise of 1960s scifi: systems you can hack using logic games and creative lies. --- CHANGELOG.md | 6 +++--- ShellCheck.cabal | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dc7f6ea..6c8beeb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ -## Git +## v0.10.0 - 2024-03-07 ### Added +- Precompiled binaries for macOS ARM64 (darwin.aarch64) - Added support for busybox sh - Added flag --rcfile to specify an rc file by name. - Added `extended-analysis=true` directive to enable/disable dataflow analysis @@ -16,8 +17,7 @@ ### Fixed - source statements with here docs now work correctly - -### Changed +- "(Array.!): undefined array element" error should no longer occur ## v0.9.0 - 2022-12-12 diff --git a/ShellCheck.cabal b/ShellCheck.cabal index a12f75e..fc52b12 100644 --- a/ShellCheck.cabal +++ b/ShellCheck.cabal @@ -1,5 +1,5 @@ Name: ShellCheck -Version: 0.9.0 +Version: 0.10.0 Synopsis: Shell script analysis tool License: GPL-3 License-file: LICENSE From 94214ee725122b91374b1782e93ae239aff04762 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 7 Mar 2024 19:11:12 -0800 Subject: [PATCH 690/763] Post-release CHANGELOG --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c8beeb..b40245a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## Git +### Added +### Fixed + + ## v0.10.0 - 2024-03-07 ### Added - Precompiled binaries for macOS ARM64 (darwin.aarch64) From 50db9a29c45f1f2a0db7ec60c8850c99e8e31d6e Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 7 Mar 2024 19:11:32 -0800 Subject: [PATCH 691/763] Check source details before git details --- test/check_release | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/test/check_release b/test/check_release index fd1dbca..4aef69a 100755 --- a/test/check_release +++ b/test/check_release @@ -12,6 +12,17 @@ then fail "There are uncommitted changes" fi +version=${current#v} +if ! grep "Version:" ShellCheck.cabal | grep -qFw "$version" +then + fail "The cabal file does not match tag version $version" +fi + +if ! grep -qF "## $current" CHANGELOG.md +then + fail "CHANGELOG.md does not contain '## $current'" +fi + current=$(git tag --points-at) if [[ -z "$current" ]] then @@ -34,17 +45,6 @@ then fail "You are not on master" fi -version=${current#v} -if ! grep "Version:" ShellCheck.cabal | grep -qFw "$version" -then - fail "The cabal file does not match tag version $version" -fi - -if ! grep -qF "## $current" CHANGELOG.md -then - fail "CHANGELOG.md does not contain '## $current'" -fi - if [[ $(git log -1 --pretty=%B) != "Stable version "* ]] then fail "Expected git log message to be 'Stable version ...'" From 9cb21c8557cb981e5f49da20af9335bb68f04dee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lawrence=20Vel=C3=A1zquez?= Date: Fri, 8 Mar 2024 18:24:08 -0500 Subject: [PATCH 692/763] Recommend `typeset` instead of `declare` in SC2324 Bash has both `typeset` and `declare`, but ksh has `typeset` only. Recommend the more portable alternative to users. --- src/ShellCheck/Analytics.hs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 1cc8bf8..f37ac1d 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -5017,7 +5017,8 @@ checkPlusEqualsNumber params t = state <- CF.getIncomingState cfga id guard $ isNumber state word guard . not $ fromMaybe False $ CF.variableMayBeDeclaredInteger state var - return $ warn id 2324 "var+=1 will append, not increment. Use (( var += 1 )), declare -i var, or quote number to silence." + -- Recommend "typeset" because ksh does not have "declare". + return $ warn id 2324 "var+=1 will append, not increment. Use (( var += 1 )), typeset -i var, or quote number to silence." _ -> return () where From 52dc66349b0882b0d6ac3d81a81f3eef0b155158 Mon Sep 17 00:00:00 2001 From: Joachim Ansorg Date: Tue, 12 Mar 2024 17:36:20 +0100 Subject: [PATCH 693/763] fix build of linux.aarch64 --- build/linux.aarch64/Dockerfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/build/linux.aarch64/Dockerfile b/build/linux.aarch64/Dockerfile index d5320e9..fadb6a4 100644 --- a/build/linux.aarch64/Dockerfile +++ b/build/linux.aarch64/Dockerfile @@ -12,11 +12,15 @@ RUN apt-get update && apt-get install -y llvm gcc-$TARGET # The rest are from 22.10 RUN sed -e 's/focal/kinetic/g' -i /etc/apt/sources.list +# Kinetic does not receive updates anymore, switch to last available +RUN sed -e 's/archive.ubuntu.com/old-releases.ubuntu.com/g' -i /etc/apt/sources.list +RUN sed -e 's/security.ubuntu.com/old-releases.ubuntu.com/g' -i /etc/apt/sources.list + RUN apt-get update && apt-get install -y ghc alex happy automake autoconf build-essential curl qemu-user-static # Build GHC WORKDIR /ghc -RUN curl -L "https://downloads.haskell.org/~ghc/9.2.5/ghc-9.2.5-src.tar.xz" | tar xJ --strip-components=1 +RUN curl -L "https://downloads.haskell.org/~ghc/9.2.8/ghc-9.2.8-src.tar.xz" | tar xJ --strip-components=1 RUN ./boot && ./configure --host x86_64-linux-gnu --build x86_64-linux-gnu --target "$TARGET" RUN cp mk/flavours/quick-cross.mk mk/build.mk && make -j "$(nproc)" RUN make install From c4123375e04931b3d0deb08728490687bc2fb3fd Mon Sep 17 00:00:00 2001 From: Joachim Ansorg Date: Tue, 12 Mar 2024 18:00:36 +0100 Subject: [PATCH 694/763] build smaller ShellCheck binary for Linux x86_64 --- build/linux.x86_64/Dockerfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/build/linux.x86_64/Dockerfile b/build/linux.x86_64/Dockerfile index 3112ac2..edafb36 100644 --- a/build/linux.x86_64/Dockerfile +++ b/build/linux.x86_64/Dockerfile @@ -1,4 +1,8 @@ -FROM alpine:latest +FROM alpine:3.16 +# alpine:3.16 (GHC 9.0.1): 5.8 megabytes +# alpine:3.17 (GHC 9.0.2): 15.0 megabytes +# alpine:3.18 (GHC 9.4.4): 29.0 megabytes +# alpine:3.19 (GHC 9.4.7): 29.0 megabytes ENV TARGETNAME linux.x86_64 From 0a7bb1822e9c80dae21aa6aaba6d8da3de5d9e94 Mon Sep 17 00:00:00 2001 From: Hugo Sousa <55895340+hugos99@users.noreply.github.com> Date: Thu, 4 Apr 2024 12:26:20 +0100 Subject: [PATCH 695/763] Update README.md to add macOS Arm64 pre-compiled binaries link --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 6542ac2..366c427 100644 --- a/README.md +++ b/README.md @@ -233,6 +233,7 @@ Alternatively, you can download pre-compiled binaries for the latest release her * [Linux, x86_64](https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.x86_64.tar.xz) (statically linked) * [Linux, armv6hf](https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.armv6hf.tar.xz), i.e. Raspberry Pi (statically linked) * [Linux, aarch64](https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.aarch64.tar.xz) aka ARM64 (statically linked) +* [macOS, aarch64](https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.darwin.aarch64.tar.xz) * [macOS, x86_64](https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.darwin.x86_64.tar.xz) * [Windows, x86](https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.zip) From 30b32af873c8b9e24731a5cb08bb20b7f148fa2d Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 4 Apr 2024 19:50:08 -0700 Subject: [PATCH 696/763] Add updating build images to release checks --- test/check_release | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/check_release b/test/check_release index 4aef69a..665b265 100755 --- a/test/check_release +++ b/test/check_release @@ -56,11 +56,14 @@ cat << EOF Manual Checklist $((i++)). Make sure none of the automated checks above failed +$((i++)). Run \`build/build_builder build/*/\` to update all builder images. +$((j++)). \`build/run_builder dist-newstyle/sdist/ShellCheck-*.tar.gz build/*/\` to verify that they work. +$((j++)). \`for f in \$(cat build/*/tag); do docker push "\$f"; done\` to upload them. +$((i++)). Run test/distrotest to ensure that most distros can build OOTB. $((i++)). Make sure GitHub Build currently passes: https://github.com/koalaman/shellcheck/actions $((i++)). Make sure SnapCraft build currently works: https://build.snapcraft.io/user/koalaman -$((i++)). Run test/distrotest to ensure that most distros can build OOTB. $((i++)). Format and read over the manual for bad formatting and outdated info. -$((i++)). Make sure the Hackage package builds. +$((i++)). Make sure the Hackage package builds locally. Release Steps From 5241878e5919d3581a8e0208c3d2345532dbb65f Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Fri, 5 Apr 2024 17:15:04 -0700 Subject: [PATCH 697/763] Update Windows build image with new cURL URL --- build/windows.x86_64/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/windows.x86_64/Dockerfile b/build/windows.x86_64/Dockerfile index 1e5c5d9..2ae78ac 100644 --- a/build/windows.x86_64/Dockerfile +++ b/build/windows.x86_64/Dockerfile @@ -12,7 +12,7 @@ WORKDIR /haskell RUN curl -L "https://downloads.haskell.org/~ghc/8.10.4/ghc-8.10.4-x86_64-unknown-mingw32.tar.xz" | tar xJ --strip-components=1 WORKDIR /haskell/bin RUN curl -L "https://downloads.haskell.org/~cabal/cabal-install-3.2.0.0/cabal-install-3.2.0.0-x86_64-unknown-mingw32.zip" | busybox unzip - -RUN curl -L "https://curl.se/windows/dl-7.84.0/curl-7.84.0-win64-mingw.zip" | busybox unzip - && mv curl-7.84.0-win64-mingw/bin/* . +RUN curl -L "https://curl.se/windows/dl-8.7.1_7/curl-8.7.1_7-win64-mingw.zip" | busybox unzip - && mv curl-*-win64-mingw/bin/* . ENV WINEPATH /haskell/bin # It's unknown whether Cabal on Windows suffers from the same issue From 04a86245a10fe0a9e48755237f315999986f54c0 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 8 Apr 2024 20:24:28 -0700 Subject: [PATCH 698/763] Remove trailing space in output (fixes #2961) --- src/ShellCheck/Formatter/TTY.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Formatter/TTY.hs b/src/ShellCheck/Formatter/TTY.hs index e503639..117da6e 100644 --- a/src/ShellCheck/Formatter/TTY.hs +++ b/src/ShellCheck/Formatter/TTY.hs @@ -169,7 +169,7 @@ showFixedString color comments lineNum fileLines = -- and/or other unrelated lines. let (excerptFix, excerpt) = sliceFile mergedFix fileLines -- in the spirit of error prone - putStrLn $ color "message" "Did you mean: " + putStrLn $ color "message" "Did you mean:" putStrLn $ unlines $ applyFix excerptFix excerpt cuteIndent :: PositionedComment -> String From 2c5155e43d030e1325c3c2765d8f492024b02fd9 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 14 Apr 2024 18:47:19 -0700 Subject: [PATCH 699/763] Warn about capturing the output of redirected commands. --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 42 +++++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b40245a..25fc688 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ## Git ### Added +- SC2327/SC2328: Warn about capturing the output of redirected commands. ### Fixed diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index f37ac1d..8b74ac6 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -204,6 +204,7 @@ nodeChecks = [ ,checkUnnecessaryArithmeticExpansionIndex ,checkUnnecessaryParens ,checkPlusEqualsNumber + ,checkExpansionWithRedirection ] optionalChecks = map fst optionalTreeChecks @@ -5040,5 +5041,46 @@ checkPlusEqualsNumber params t = isUnquotedNumber || isNumericalVariableName || isNumericalVariableExpansion + +prop_checkExpansionWithRedirection1 = verify checkExpansionWithRedirection "var=$(foo > bar)" +prop_checkExpansionWithRedirection2 = verify checkExpansionWithRedirection "var=`foo 1> bar`" +prop_checkExpansionWithRedirection3 = verify checkExpansionWithRedirection "var=${ foo >> bar; }" +prop_checkExpansionWithRedirection4 = verify checkExpansionWithRedirection "var=$(foo | bar > baz)" +prop_checkExpansionWithRedirection5 = verifyNot checkExpansionWithRedirection "stderr=$(foo 2>&1 > /dev/null)" +prop_checkExpansionWithRedirection6 = verifyNot checkExpansionWithRedirection "var=$(foo; bar > baz)" +prop_checkExpansionWithRedirection7 = verifyNot checkExpansionWithRedirection "var=$(foo > bar; baz)" +prop_checkExpansionWithRedirection8 = verifyNot checkExpansionWithRedirection "var=$(cat <&3)" +checkExpansionWithRedirection params t = + case t of + T_DollarExpansion id [cmd] -> check id cmd + T_Backticked id [cmd] -> check id cmd + T_DollarBraceCommandExpansion id [cmd] -> check id cmd + _ -> return () + where + check id pipe = + case pipe of + (T_Pipeline _ _ t@(_:_)) -> checkCmd id (last t) + _ -> return () + + checkCmd captureId (T_Redirecting _ redirs _) = walk captureId redirs + + walk captureId [] = return () + walk captureId (t:rest) = + case t of + T_FdRedirect _ _ (T_IoDuplicate _ _ "1") -> return () + T_FdRedirect id "1" (T_IoDuplicate _ _ _) -> return () + T_FdRedirect id "" (T_IoDuplicate _ op _) | op `elem` [T_GREATAND (Id 0), T_Greater (Id 0)] -> emit id captureId True + T_FdRedirect id str (T_IoFile _ op file) | str `elem` ["", "1"] && op `elem` [ T_DGREAT (Id 0), T_Greater (Id 0) ] -> + if getLiteralString file == Just "/dev/null" + then emit id captureId False + else emit id captureId True + _ -> walk captureId rest + + emit redirectId captureId suggestTee = do + warn captureId 2327 "This command substitution will be empty because the command's output gets redirected away." + err redirectId 2328 $ "This redirection takes output away from the command substitution" ++ if suggestTee then " (use tee to duplicate)." else "." + + + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) From 69fe4e1306572fa639d92e80afe2639d6c12247f Mon Sep 17 00:00:00 2001 From: Syuugo Date: Thu, 25 Apr 2024 10:35:43 +0900 Subject: [PATCH 700/763] Upgrade build workflow dependencies --- .github/workflows/build.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c49b25a..378a0cf 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,7 +15,7 @@ jobs: sudo apt-get install cabal-install - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 @@ -37,7 +37,7 @@ jobs: mv dist-newstyle/sdist/*.tar.gz source/source.tar.gz - name: Upload artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: source path: source/ @@ -51,10 +51,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Download artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 - name: Build source run: | @@ -63,7 +63,7 @@ jobs: ( cd bin && ../build/run_builder ../source/source.tar.gz ../build/${{matrix.build}} ) - name: Upload artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: bin path: bin/ @@ -74,10 +74,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Download artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 - name: Work around GitHub permissions bug run: chmod +x bin/*/shellcheck* @@ -92,7 +92,7 @@ jobs: rm -rf */ README* LICENSE* - name: Upload artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: deploy path: deploy/ @@ -109,10 +109,10 @@ jobs: sudo apt-get install hub - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Download artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 - name: Upload to GitHub env: From 796c6bd8482e4666c4f73d2874c3949c2bed801e Mon Sep 17 00:00:00 2001 From: Jan Dubois Date: Wed, 24 Apr 2024 19:05:29 -0700 Subject: [PATCH 701/763] Add new bats variables stderr and stderr_lines These are being set by `run --separate-stderr` and have been introduced in https://github.com/bats-core/bats-core/releases/tag/v1.5.0 --- src/ShellCheck/AnalyzerLib.hs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index d265ace..a2d61d2 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -535,7 +535,9 @@ getModifiedVariables t = T_BatsTest {} -> [ (t, t, "lines", DataArray SourceExternal), (t, t, "status", DataString SourceInteger), - (t, t, "output", DataString SourceExternal) + (t, t, "output", DataString SourceExternal), + (t, t, "stderr", DataString SourceExternal), + (t, t, "stderr_lines", DataArray SourceExternal) ] -- Count [[ -v foo ]] as an "assignment". From 4f81dbe839091a06a5cfaea695cf1c451ff07565 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 4 May 2024 14:21:12 -0700 Subject: [PATCH 702/763] Add warning about uninvoked functions, reduce repeated triggering of SC2317 (fixes #2966) --- CHANGELOG.md | 2 ++ src/ShellCheck/Analytics.hs | 23 ++++++++++++++++++++--- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 25fc688..5277893 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,9 @@ ## Git ### Added - SC2327/SC2328: Warn about capturing the output of redirected commands. +- SC2329: Warn when (non-escaping) functions are never invoked. ### Fixed +- SC2317 about unreachable commands is now less spammy for nested ones. ## v0.10.0 - 2024-03-07 diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 8b74ac6..685fbf4 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -4896,16 +4896,33 @@ checkBatsTestDoesNotUseNegation params t = prop_checkCommandIsUnreachable1 = verify checkCommandIsUnreachable "foo; bar; exit; baz" prop_checkCommandIsUnreachable2 = verify checkCommandIsUnreachable "die() { exit; }; foo; bar; die; baz" prop_checkCommandIsUnreachable3 = verifyNot checkCommandIsUnreachable "foo; bar || exit; baz" +prop_checkCommandIsUnreachable4 = verifyNot checkCommandIsUnreachable "f() { foo; }; # Maybe sourced" +prop_checkCommandIsUnreachable5 = verify checkCommandIsUnreachable "f() { foo; }; exit # Not sourced" checkCommandIsUnreachable params t = case t of T_Pipeline {} -> sequence_ $ do cfga <- cfgAnalysis params - state <- CF.getIncomingState cfga id + state <- CF.getIncomingState cfga (getId t) guard . not $ CF.stateIsReachable state guard . not $ isSourced params t - return $ info id 2317 "Command appears to be unreachable. Check usage (or ignore if invoked indirectly)." + guard . not $ any (\t -> isUnreachable t || isUnreachableFunction t) $ NE.drop 1 $ getPath (parentMap params) t + return $ info (getId t) 2317 "Command appears to be unreachable. Check usage (or ignore if invoked indirectly)." + T_Function id _ _ _ _ -> + when (isUnreachableFunction t + && (not . any isUnreachableFunction . NE.drop 1 $ getPath (parentMap params) t) + && (not $ isSourced params t)) $ + info id 2329 "This function is never invoked. Check usage (or ignored if invoked indirectly)." _ -> return () - where id = getId t + where + isUnreachableFunction :: Token -> Bool + isUnreachableFunction f = + case f of + T_Function id _ _ _ t -> isUnreachable t + _ -> False + isUnreachable t = fromMaybe False $ do + cfga <- cfgAnalysis params + state <- CF.getIncomingState cfga (getId t) + return . not $ CF.stateIsReachable state prop_checkOverwrittenExitCode1 = verify checkOverwrittenExitCode "x; [ $? -eq 1 ] || [ $? -eq 2 ]" From 76ff702e9385215a888ed21bf4330f614ab6c355 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 4 May 2024 15:12:13 -0700 Subject: [PATCH 703/763] Supress SC2015 about `A && B || C` when B is a test. --- CHANGELOG.md | 2 ++ src/ShellCheck/Analytics.hs | 16 +++++----------- src/ShellCheck/AnalyzerLib.hs | 8 ++++++++ 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5277893..fe38b8a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ### Added - SC2327/SC2328: Warn about capturing the output of redirected commands. - SC2329: Warn when (non-escaping) functions are never invoked. +### Changed +- SC2015 about `A && B || C` no longer triggers when B is a test command. ### Fixed - SC2317 about unreachable commands is now less spammy for nested ones. diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 685fbf4..175dea6 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -877,8 +877,9 @@ prop_checkShorthandIf5 = verifyNot checkShorthandIf "foo && rm || printf b" prop_checkShorthandIf6 = verifyNot checkShorthandIf "if foo && bar || baz; then true; fi" prop_checkShorthandIf7 = verifyNot checkShorthandIf "while foo && bar || baz; do true; done" prop_checkShorthandIf8 = verify checkShorthandIf "if true; then foo && bar || baz; fi" -checkShorthandIf params x@(T_OrIf _ (T_AndIf id _ _) (T_Pipeline _ _ t)) - | not (isOk t || inCondition) = +prop_checkShorthandIf9 = verifyNot checkShorthandIf "foo && [ -x /file ] || bar" +checkShorthandIf params x@(T_OrIf _ (T_AndIf id _ b) (T_Pipeline _ _ t)) + | not (isOk t || inCondition) && not (isTestCommand b) = info id 2015 "Note that A && B || C is not if-then-else. C may run when A is true." where isOk [t] = isAssignment t || fromMaybe False (do @@ -4197,7 +4198,7 @@ checkBadTestAndOr params t = in mapM_ checkTest commandWithSeps checkTest (before, cmd, after) = - when (isTest cmd) $ do + when (isTestCommand cmd) $ do checkPipe before checkPipe after @@ -4213,17 +4214,10 @@ checkBadTestAndOr params t = T_AndIf _ _ rhs -> checkAnds id rhs T_OrIf _ _ rhs -> checkAnds id rhs T_Pipeline _ _ list | not (null list) -> checkAnds id (last list) - cmd -> when (isTest cmd) $ + cmd -> when (isTestCommand cmd) $ errWithFix id 2265 "Use && for logical AND. Single & will background and return true." $ (fixWith [replaceEnd id params 0 "&"]) - isTest t = - case t of - T_Condition {} -> True - T_SimpleCommand {} -> t `isCommand` "test" - T_Redirecting _ _ t -> isTest t - T_Annotation _ _ t -> isTest t - _ -> False prop_checkComparisonWithLeadingX1 = verify checkComparisonWithLeadingX "[ x$foo = xlol ]" prop_checkComparisonWithLeadingX2 = verify checkComparisonWithLeadingX "test x$foo = xlol" diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index d265ace..c6e4e14 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -929,6 +929,14 @@ modifiesVariable params token name = Assignment (_, _, n, source) -> isTrueAssignmentSource source && n == name _ -> False +isTestCommand t = + case t of + T_Condition {} -> True + T_SimpleCommand {} -> t `isCommand` "test" + T_Redirecting _ _ t -> isTestCommand t + T_Annotation _ _ t -> isTestCommand t + T_Pipeline _ _ [t] -> isTestCommand t + _ -> False return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) From d705716dc45d58eef51231292758e2af9e3da30b Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 4 May 2024 15:22:09 -0700 Subject: [PATCH 704/763] Account for annotations in SC2215. Fixes #2975. --- src/ShellCheck/Analytics.hs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 175dea6..dcb4ce8 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -4000,6 +4000,7 @@ prop_checkUselessBang6 = verify checkUselessBang "set -e; { ! true; }" prop_checkUselessBang7 = verifyNot checkUselessBang "set -e; x() { ! [ x ]; }" prop_checkUselessBang8 = verifyNot checkUselessBang "set -e; if { ! true; }; then true; fi" prop_checkUselessBang9 = verifyNot checkUselessBang "set -e; while ! true; do true; done" +prop_checkUselessBang10 = verify checkUselessBang "set -e\nshellcheck disable=SC0000\n! true\nrest" checkUselessBang params t = when (hasSetE params) $ mapM_ check (getNonReturningCommands t) where check t = @@ -4008,6 +4009,7 @@ checkUselessBang params t = when (hasSetE params) $ mapM_ check (getNonReturning addComment $ makeCommentWithFix InfoC id 2251 "This ! is not on a condition and skips errexit. Use `&& exit 1` instead, or make sure $? is checked." (fixWith [replaceStart id params 1 "", replaceEnd (getId cmd) params 0 " && exit 1"]) + T_Annotation _ _ t -> check t _ -> return () -- Get all the subcommands that aren't likely to be the return value From a7a906e2cbca41611f232208ae062fe7ddc719f7 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 4 May 2024 16:28:56 -0700 Subject: [PATCH 705/763] Allow SC2154 to trigger in arrays (fixes #2970) --- src/ShellCheck/Analytics.hs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index dcb4ce8..dc84a78 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2447,6 +2447,7 @@ prop_checkUnassignedReferences_minusZDefault = verifyNotTree checkUnassignedRefe prop_checkUnassignedReferences50 = verifyNotTree checkUnassignedReferences "echo ${foo:+bar}" prop_checkUnassignedReferences51 = verifyNotTree checkUnassignedReferences "echo ${foo:+$foo}" prop_checkUnassignedReferences52 = verifyNotTree checkUnassignedReferences "wait -p pid; echo $pid" +prop_checkUnassignedReferences53 = verify checkUnassignedReferences "x=($foo)" checkUnassignedReferences = checkUnassignedReferences' False checkUnassignedReferences' includeGlobals params t = warnings @@ -2502,14 +2503,12 @@ checkUnassignedReferences' includeGlobals params t = warnings warnings = execWriter . sequence $ mapMaybe warningFor unassigned - -- Due to parsing, foo=( [bar]=baz ) parses 'bar' as a reference even for assoc arrays. - -- Similarly, ${foo[bar baz]} may not be referencing bar/baz. Just skip these. + -- ${foo[bar baz]} may not be referencing bar/baz. Just skip these. -- We can also have ${foo:+$foo} should be treated like [[ -n $foo ]] && echo $foo isException var t = any shouldExclude $ getPath (parentMap params) t where shouldExclude t = case t of - T_Array {} -> True (T_DollarBraced _ _ l) -> let str = concat $ oversimplify l ref = getBracedReference str From a13cb85f49c074ce6ab4644beaf8665a3ff6f395 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 4 May 2024 16:34:21 -0700 Subject: [PATCH 706/763] Fixed broken test due to bad build cache --- src/ShellCheck/Analytics.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index dc84a78..f5e57e2 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2447,7 +2447,7 @@ prop_checkUnassignedReferences_minusZDefault = verifyNotTree checkUnassignedRefe prop_checkUnassignedReferences50 = verifyNotTree checkUnassignedReferences "echo ${foo:+bar}" prop_checkUnassignedReferences51 = verifyNotTree checkUnassignedReferences "echo ${foo:+$foo}" prop_checkUnassignedReferences52 = verifyNotTree checkUnassignedReferences "wait -p pid; echo $pid" -prop_checkUnassignedReferences53 = verify checkUnassignedReferences "x=($foo)" +prop_checkUnassignedReferences53 = verifyTree checkUnassignedReferences "x=($foo)" checkUnassignedReferences = checkUnassignedReferences' False checkUnassignedReferences' includeGlobals params t = warnings From ac8fb00504ed6da83fe5c5f83e72e4663ff6b439 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 4 May 2024 16:45:52 -0700 Subject: [PATCH 707/763] Account for BusyBox support of [[ ]] (fixes #2967) --- CHANGELOG.md | 2 ++ src/ShellCheck/Analytics.hs | 11 ++++++++--- src/ShellCheck/AnalyzerLib.hs | 10 ---------- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fe38b8a..8ae176d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,10 +2,12 @@ ### Added - SC2327/SC2328: Warn about capturing the output of redirected commands. - SC2329: Warn when (non-escaping) functions are never invoked. +- SC2330: Warn about unsupported glob matches with [[ .. ]] in BusyBox. ### Changed - SC2015 about `A && B || C` no longer triggers when B is a test command. ### Fixed - SC2317 about unreachable commands is now less spammy for nested ones. +- SC2292, optional suggestion for [[ ]], now triggers for Busybox. ## v0.10.0 - 2024-03-07 diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index f5e57e2..a89f940 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1525,6 +1525,7 @@ prop_checkComparisonAgainstGlob3 = verify checkComparisonAgainstGlob "[ $cow = * prop_checkComparisonAgainstGlob4 = verifyNot checkComparisonAgainstGlob "[ $cow = foo ]" prop_checkComparisonAgainstGlob5 = verify checkComparisonAgainstGlob "[[ $cow != $bar ]]" prop_checkComparisonAgainstGlob6 = verify checkComparisonAgainstGlob "[ $f != /* ]" +prop_checkComparisonAgainstGlob7 = verify checkComparisonAgainstGlob "#!/bin/busybox sh\n[[ $f == *foo* ]]" checkComparisonAgainstGlob _ (TC_Binary _ DoubleBracket op _ (T_NormalWord id [T_DollarBraced _ _ _])) | op `elem` ["=", "==", "!="] = warn id 2053 $ "Quote the right-hand side of " ++ op ++ " in [[ ]] to prevent glob matching." @@ -1532,10 +1533,14 @@ checkComparisonAgainstGlob params (TC_Binary _ SingleBracket op _ word) | op `elem` ["=", "==", "!="] && isGlob word = err (getId word) 2081 msg where - msg = if isBashLike params + msg = if (shellType params) `elem` [Bash, Ksh] -- Busybox does not support glob matching then "[ .. ] can't match globs. Use [[ .. ]] or case statement." else "[ .. ] can't match globs. Use a case statement." +checkComparisonAgainstGlob params (TC_Binary _ DoubleBracket op _ word) + | shellType params == BusyboxSh && op `elem` ["=", "==", "!="] && isGlob word = + err (getId word) 2330 "BusyBox [[ .. ]] does not support glob matching. Use a case statement." + checkComparisonAgainstGlob _ _ = return () prop_checkCaseAgainstGlob1 = verify checkCaseAgainstGlob "case foo in lol$n) foo;; esac" @@ -4534,13 +4539,13 @@ prop_checkRequireDoubleBracket2 = verifyTree checkRequireDoubleBracket "[ foo -o prop_checkRequireDoubleBracket3 = verifyNotTree checkRequireDoubleBracket "#!/bin/sh\n[ -x foo ]" prop_checkRequireDoubleBracket4 = verifyNotTree checkRequireDoubleBracket "[[ -x foo ]]" checkRequireDoubleBracket params = - if isBashLike params + if (shellType params) `elem` [Bash, Ksh, BusyboxSh] then nodeChecksToTreeCheck [check] params else const [] where check _ t = case t of T_Condition id SingleBracket _ -> - styleWithFix id 2292 "Prefer [[ ]] over [ ] for tests in Bash/Ksh." (fixFor t) + styleWithFix id 2292 "Prefer [[ ]] over [ ] for tests in Bash/Ksh/Busybox." (fixFor t) _ -> return () fixFor t = fixWith $ diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index c6e4e14..cae73b1 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -902,16 +902,6 @@ supportsArrays Bash = True supportsArrays Ksh = True supportsArrays _ = False --- Returns true if the shell is Bash or Ksh (sorry for the name, Ksh) -isBashLike :: Parameters -> Bool -isBashLike params = - case shellType params of - Bash -> True - Ksh -> True - Dash -> False - BusyboxSh -> False - Sh -> False - isTrueAssignmentSource c = case c of DataString SourceChecked -> False From 78d1ee0222a114597abba1ca8f0784b673bf7d97 Mon Sep 17 00:00:00 2001 From: Bryan Honof Date: Fri, 24 May 2024 17:15:09 +0200 Subject: [PATCH 708/763] Add Flox to list of installation methods --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 366c427..754f897 100644 --- a/README.md +++ b/README.md @@ -228,6 +228,11 @@ Using the [nix package manager](https://nixos.org/nix): nix-env -iA nixpkgs.shellcheck ``` +Using the [Flox package manager](https://flox.dev/) +```sh +flox install shellcheck +``` + Alternatively, you can download pre-compiled binaries for the latest release here: * [Linux, x86_64](https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.x86_64.tar.xz) (statically linked) From 15de97e33f462434272f13e1ec3aa9cd5387441b Mon Sep 17 00:00:00 2001 From: Meng Zhuo Date: Thu, 30 May 2024 19:20:21 +0800 Subject: [PATCH 709/763] Add linux.riscv64 precompiled support --- .github/workflows/build.yml | 2 +- .multi_arch_docker | 1 + CHANGELOG.md | 1 + build/linux.riscv64/Dockerfile | 47 ++++++++++++++++++++++++++++++++++ build/linux.riscv64/build | 15 +++++++++++ build/linux.riscv64/tag | 1 + 6 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 build/linux.riscv64/Dockerfile create mode 100755 build/linux.riscv64/build create mode 100644 build/linux.riscv64/tag diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c49b25a..e30b2ac 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -47,7 +47,7 @@ jobs: needs: package_source strategy: matrix: - build: [linux.x86_64, linux.aarch64, linux.armv6hf, darwin.x86_64, darwin.aarch64, windows.x86_64] + build: [linux.x86_64, linux.aarch64, linux.armv6hf, linux.riscv64, darwin.x86_64, darwin.aarch64, windows.x86_64] runs-on: ubuntu-latest steps: - name: Checkout repository diff --git a/.multi_arch_docker b/.multi_arch_docker index 1c5d32b..81048a2 100755 --- a/.multi_arch_docker +++ b/.multi_arch_docker @@ -80,6 +80,7 @@ function multi_arch_docker::main() { export DOCKER_PLATFORMS='linux/amd64' DOCKER_PLATFORMS+=' linux/arm64' DOCKER_PLATFORMS+=' linux/arm/v6' + DOCKER_PLATFORMS+=' linux/riscv64' multi_arch_docker::install_docker_buildx multi_arch_docker::login_to_docker_hub diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ae176d..43db00c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ - SC2327/SC2328: Warn about capturing the output of redirected commands. - SC2329: Warn when (non-escaping) functions are never invoked. - SC2330: Warn about unsupported glob matches with [[ .. ]] in BusyBox. +- Precompiled binaries for Linux riscv64 (linux.riscv64) ### Changed - SC2015 about `A && B || C` no longer triggers when B is a test command. ### Fixed diff --git a/build/linux.riscv64/Dockerfile b/build/linux.riscv64/Dockerfile new file mode 100644 index 0000000..648ef2d --- /dev/null +++ b/build/linux.riscv64/Dockerfile @@ -0,0 +1,47 @@ +FROM ubuntu:22.04 + +ENV TARGETNAME linux.riscv64 +ENV TARGET riscv64-linux-gnu + +USER root +ENV DEBIAN_FRONTEND noninteractive + +# Init base +RUN apt update -y + +# Install qemu +RUN apt install -y --no-install-recommends build-essential ninja-build python3 pkg-config libglib2.0-dev libpixman-1-dev curl ca-certificates python3-virtualenv +WORKDIR /qemu +RUN curl -Lv "https://download.qemu.org/qemu-9.0.0.tar.xz" | tar xJ --strip-components=1 +RUN ./configure --target-list=riscv64-linux-user --static --disable-system --disable-pie +RUN cd build && ninja qemu-riscv64 +ENV QEMU_EXECVE 1 + + +# Set up a riscv64 userspace +RUN apt install -y --no-install-recommends debootstrap +RUN debootstrap --arch=riscv64 --foreign jammy /rvfs http://ports.ubuntu.com/ubuntu-ports +RUN cp /qemu/build/qemu-riscv64 /rvfs/usr/bin/qemu-riscv64-static + +RUN printf > /bin/rv '%s\n' '#!/bin/sh' 'chroot /rvfs /usr/bin/qemu-riscv64-static /usr/bin/env "$@"' +RUN chmod +x /bin/rv +RUN [ ! -e /rvfs/debootstrap ] || rv '/debootstrap/debootstrap' --second-stage + +# Install deps in the chroot +RUN printf > /rvfs/etc/apt/sources.list '%s\n' 'deb http://ports.ubuntu.com/ubuntu-ports jammy main universe' +RUN rv apt update -y +RUN rv apt install -y --no-install-recommends ghc cabal-install + +# Finally we can build the current dependencies. This takes hours. +# jobs must be 1, GHS riscv will use about 40G memory +RUN rv cabal update +RUN IFS=";" && rv cabal install --dependencies-only --jobs=1 ShellCheck +RUN IFS=';' && rv cabal install --lib --jobs=1 fgl + +# Clean up +RUN rm -rf /qemu + +# Copy the build script +WORKDIR /rvfs/scratch +COPY build /rvfs/usr/bin/build +ENTRYPOINT ["/bin/rv", "/usr/bin/build"] diff --git a/build/linux.riscv64/build b/build/linux.riscv64/build new file mode 100755 index 0000000..19cb143 --- /dev/null +++ b/build/linux.riscv64/build @@ -0,0 +1,15 @@ +#!/bin/sh +set -xe +{ + tar xzv --strip-components=1 + chmod +x striptests && ./striptests + mkdir "$TARGETNAME" + cabal update + ( IFS=';'; cabal build --enable-executable-static ) + find . -name shellcheck -type f -exec mv {} "$TARGETNAME/" \; + ls -l "$TARGETNAME" + "$TARGET-strip" -s "$TARGETNAME/shellcheck" + ls -l "$TARGETNAME" + qemu-riscv64-static "$TARGETNAME/shellcheck" --version +} >&2 +tar czv "$TARGETNAME" diff --git a/build/linux.riscv64/tag b/build/linux.riscv64/tag new file mode 100644 index 0000000..901eaaa --- /dev/null +++ b/build/linux.riscv64/tag @@ -0,0 +1 @@ +koalaman/scbuilder-linux-riscv64 From 23e76de4f2a640ecd7c8c1da23495d985ef095e6 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 11 Jun 2024 06:00:22 +0000 Subject: [PATCH 710/763] Allow riscv64 image to run without binfmt_misc --- build/linux.riscv64/Dockerfile | 44 ++++++----- build/linux.riscv64/build | 10 ++- build/linux.riscv64/cabal.project.freeze | 93 ++++++++++++++++++++++++ 3 files changed, 126 insertions(+), 21 deletions(-) create mode 100644 build/linux.riscv64/cabal.project.freeze diff --git a/build/linux.riscv64/Dockerfile b/build/linux.riscv64/Dockerfile index 648ef2d..0ac95e5 100644 --- a/build/linux.riscv64/Dockerfile +++ b/build/linux.riscv64/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:22.04 +FROM ubuntu:24.04 ENV TARGETNAME linux.riscv64 ENV TARGET riscv64-linux-gnu @@ -7,39 +7,47 @@ USER root ENV DEBIAN_FRONTEND noninteractive # Init base -RUN apt update -y +RUN apt-get update -y # Install qemu -RUN apt install -y --no-install-recommends build-essential ninja-build python3 pkg-config libglib2.0-dev libpixman-1-dev curl ca-certificates python3-virtualenv +RUN apt-get install -y --no-install-recommends build-essential ninja-build python3 pkg-config libglib2.0-dev libpixman-1-dev curl ca-certificates python3-virtualenv git python3-setuptools WORKDIR /qemu -RUN curl -Lv "https://download.qemu.org/qemu-9.0.0.tar.xz" | tar xJ --strip-components=1 -RUN ./configure --target-list=riscv64-linux-user --static --disable-system --disable-pie +RUN git clone --depth 1 https://github.com/koalaman/qemu . +#RUN git clone https://github.com/balena-io/qemu . +# Release 7.0.0 +#RUN git checkout 639d1d8903f65d74eb04c49e0df7a4b2f014cd86 +RUN ./configure --target-list=riscv64-linux-user --static --disable-system --disable-pie --disable-werror RUN cd build && ninja qemu-riscv64 ENV QEMU_EXECVE 1 # Set up a riscv64 userspace -RUN apt install -y --no-install-recommends debootstrap -RUN debootstrap --arch=riscv64 --foreign jammy /rvfs http://ports.ubuntu.com/ubuntu-ports +RUN apt-get install -y --no-install-recommends debootstrap +RUN debootstrap --arch=riscv64 --foreign noble /rvfs http://ports.ubuntu.com/ubuntu-ports RUN cp /qemu/build/qemu-riscv64 /rvfs/usr/bin/qemu-riscv64-static -RUN printf > /bin/rv '%s\n' '#!/bin/sh' 'chroot /rvfs /usr/bin/qemu-riscv64-static /usr/bin/env "$@"' +# Command to run riscv binaries in the chroot. The Haskell runtime allocates 1TB +# vspace up front and QEmu has a RAM cost per vspace, so use ulimit to allocate +# less and reduce RAM usage. +RUN printf > /bin/rv '%s\n' '#!/bin/sh' 'ulimit -v $((10*1024*1024)); chroot /rvfs /usr/bin/qemu-riscv64-static /usr/bin/env "$@"' RUN chmod +x /bin/rv RUN [ ! -e /rvfs/debootstrap ] || rv '/debootstrap/debootstrap' --second-stage # Install deps in the chroot -RUN printf > /rvfs/etc/apt/sources.list '%s\n' 'deb http://ports.ubuntu.com/ubuntu-ports jammy main universe' -RUN rv apt update -y -RUN rv apt install -y --no-install-recommends ghc cabal-install +RUN printf > /rvfs/etc/apt/sources.list '%s\n' 'deb http://ports.ubuntu.com/ubuntu-ports noble main universe' +RUN rv apt-get update -y +RUN rv apt-get install -y --no-install-recommends ghc cabal-install -# Finally we can build the current dependencies. This takes hours. -# jobs must be 1, GHS riscv will use about 40G memory RUN rv cabal update -RUN IFS=";" && rv cabal install --dependencies-only --jobs=1 ShellCheck -RUN IFS=';' && rv cabal install --lib --jobs=1 fgl - -# Clean up -RUN rm -rf /qemu +# Generated with: cabal freeze -c 'hashable -arch-native'. We put it in /etc so cabal won't find it. +COPY cabal.project.freeze /rvfs/etc +# Awful hack to install everything from the freeze file +# This basically turns 'any.tagged ==0.8.8' into tagged-0.8.8 to install by version, +# and adds a -c before 'hashable -arch-native +integer-gmp' to make it a flag constraint. +RUN < /rvfs/etc/cabal.project.freeze sed 's/constraints:/&\n /' | grep -vw rts | sed -n -e 's/^ *\([^,]*\).*/\1/p' | sed -e 's/any\.\([^ ]*\) ==\(.*\)/\1-\2/; te; s/.*/-c\n&/; :e' > /tmp/preinstall-flags +# Finally we can build the current dependencies. This takes hours. +# There's apparently a random segfault during assembly, so retry a few times. +RUN set -x; IFS=${IFS# }; f() { rv cabal install --keep-going $(cat /tmp/preinstall-flags); }; for i in $(seq 5); do f; ret=$?; [ $ret = 0 ] && break; done; exit $ret # Copy the build script WORKDIR /rvfs/scratch diff --git a/build/linux.riscv64/build b/build/linux.riscv64/build index 19cb143..63c487f 100755 --- a/build/linux.riscv64/build +++ b/build/linux.riscv64/build @@ -1,15 +1,19 @@ #!/bin/sh set -xe +IFS=';' { tar xzv --strip-components=1 chmod +x striptests && ./striptests + # Use a freeze file to ensure we use the same dependencies we cached during + # the docker image build. We don't want to spend time compiling anything new. + cp /etc/cabal.project.freeze . mkdir "$TARGETNAME" - cabal update - ( IFS=';'; cabal build --enable-executable-static ) + # Retry in case of random segfault + cabal build --enable-executable-static || cabal build --enable-executable-static find . -name shellcheck -type f -exec mv {} "$TARGETNAME/" \; ls -l "$TARGETNAME" "$TARGET-strip" -s "$TARGETNAME/shellcheck" ls -l "$TARGETNAME" - qemu-riscv64-static "$TARGETNAME/shellcheck" --version + "$TARGETNAME/shellcheck" --version } >&2 tar czv "$TARGETNAME" diff --git a/build/linux.riscv64/cabal.project.freeze b/build/linux.riscv64/cabal.project.freeze new file mode 100644 index 0000000..cbb42e1 --- /dev/null +++ b/build/linux.riscv64/cabal.project.freeze @@ -0,0 +1,93 @@ +active-repositories: hackage.haskell.org:merge +constraints: any.Diff ==0.5, + any.OneTuple ==0.4.2, + any.QuickCheck ==2.14.3, + QuickCheck -old-random +templatehaskell, + any.StateVar ==1.2.2, + any.aeson ==2.2.3.0, + aeson +ordered-keymap, + any.array ==0.5.4.0, + any.assoc ==1.1.1, + assoc -tagged, + any.base ==4.17.2.0, + any.base-orphans ==0.9.2, + any.bifunctors ==5.6.2, + bifunctors +tagged, + any.binary ==0.8.9.1, + any.bytestring ==0.11.5.2, + any.character-ps ==0.1, + any.comonad ==5.0.8, + comonad +containers +distributive +indexed-traversable, + any.containers ==0.6.7, + any.contravariant ==1.5.5, + contravariant +semigroups +statevar +tagged, + any.data-fix ==0.3.3, + any.deepseq ==1.4.8.0, + any.directory ==1.3.7.1, + any.distributive ==0.6.2.1, + distributive +semigroups +tagged, + any.dlist ==1.0, + dlist -werror, + any.exceptions ==0.10.5, + any.fgl ==5.8.2.0, + fgl +containers042, + any.filepath ==1.4.2.2, + any.foldable1-classes-compat ==0.1, + foldable1-classes-compat +tagged, + any.generically ==0.1.1, + any.ghc-bignum ==1.3, + any.ghc-boot-th ==9.4.7, + any.ghc-prim ==0.9.1, + any.hashable ==1.4.6.0, + hashable -arch-native +integer-gmp -random-initial-seed, + any.indexed-traversable ==0.1.4, + any.indexed-traversable-instances ==0.1.2, + any.integer-conversion ==0.1.1, + any.integer-logarithms ==1.0.3.1, + integer-logarithms -check-bounds +integer-gmp, + any.mtl ==2.2.2, + any.network-uri ==2.6.4.2, + any.os-string ==2.0.3, + any.parsec ==3.1.16.1, + any.pretty ==1.1.3.6, + any.primitive ==0.9.0.0, + any.process ==1.6.17.0, + any.random ==1.2.1.2, + any.regex-base ==0.94.0.2, + any.regex-tdfa ==1.3.2.2, + regex-tdfa +doctest -force-o2, + any.rts ==1.0.2, + any.scientific ==0.3.8.0, + scientific -integer-simple, + any.semialign ==1.3.1, + semialign +semigroupoids, + any.semigroupoids ==6.0.1, + semigroupoids +comonad +containers +contravariant +distributive +tagged +unordered-containers, + any.splitmix ==0.1.0.5, + splitmix -optimised-mixer, + any.stm ==2.5.1.0, + any.strict ==0.5, + any.tagged ==0.8.8, + tagged +deepseq +transformers, + any.template-haskell ==2.19.0.0, + any.text ==2.0.2, + any.text-iso8601 ==0.1.1, + any.text-short ==0.1.6, + text-short -asserts, + any.th-abstraction ==0.7.0.0, + any.th-compat ==0.1.5, + any.these ==1.2.1, + any.time ==1.12.2, + any.time-compat ==1.9.7, + any.transformers ==0.5.6.2, + any.transformers-compat ==0.7.2, + transformers-compat -five +five-three -four +generic-deriving +mtl -three -two, + any.unix ==2.7.3, + any.unordered-containers ==0.2.20, + unordered-containers -debug, + any.uuid-types ==1.0.6, + any.vector ==0.13.1.0, + vector +boundschecks -internalchecks -unsafechecks -wall, + any.vector-stream ==0.1.0.1, + any.witherable ==0.5 +index-state: hackage.haskell.org 2024-06-17T00:48:51Z From 3946cbd4a0b477700172c04be7baa3f43777bc99 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 24 Jun 2024 05:12:21 +0000 Subject: [PATCH 711/763] Upgrade docker build images --- build/README.md | 4 + build/darwin.aarch64/build | 1 - build/darwin.x86_64/build | 1 - build/linux.aarch64/Dockerfile | 2 +- build/linux.aarch64/build | 1 - build/linux.armv6hf/Dockerfile | 64 ++++++---------- build/linux.armv6hf/build | 3 +- build/linux.armv6hf/cabal.project.freeze | 93 ++++++++++++++++++++++++ build/linux.armv6hf/scutil | 48 ++++++++++++ build/linux.riscv64/Dockerfile | 49 +++++-------- build/linux.riscv64/build | 4 +- build/windows.x86_64/build | 1 - 12 files changed, 194 insertions(+), 77 deletions(-) create mode 100644 build/linux.armv6hf/cabal.project.freeze create mode 100644 build/linux.armv6hf/scutil diff --git a/build/README.md b/build/README.md index eb745a0..31e8607 100644 --- a/build/README.md +++ b/build/README.md @@ -11,3 +11,7 @@ This makes it simple to build any release without exotic hardware or software. An image can be built and tagged using `build_builder`, and run on a source tarball using `run_builder`. + +Tip: Are you developing an image that relies on QEmu usermode emulation? + It's easy to accidentally depend on binfmt\_misc on the host OS. + Do a `echo 0 | sudo tee /proc/sys/fs/binfmt_misc/status` before testing. diff --git a/build/darwin.aarch64/build b/build/darwin.aarch64/build index 4235d3a..ff522ff 100755 --- a/build/darwin.aarch64/build +++ b/build/darwin.aarch64/build @@ -4,7 +4,6 @@ set -xe tar xzv --strip-components=1 chmod +x striptests && ./striptests mkdir "$TARGETNAME" - cabal update ( IFS=';'; cabal build $CABALOPTS ) find . -name shellcheck -type f -exec mv {} "$TARGETNAME/" \; ls -l "$TARGETNAME" diff --git a/build/darwin.x86_64/build b/build/darwin.x86_64/build index 53857e8..058cece 100755 --- a/build/darwin.x86_64/build +++ b/build/darwin.x86_64/build @@ -4,7 +4,6 @@ set -xe tar xzv --strip-components=1 chmod +x striptests && ./striptests mkdir "$TARGETNAME" - cabal update ( IFS=';'; cabal build $CABALOPTS ) find . -name shellcheck -type f -exec mv {} "$TARGETNAME/" \; ls -l "$TARGETNAME" diff --git a/build/linux.aarch64/Dockerfile b/build/linux.aarch64/Dockerfile index fadb6a4..1ffe1bd 100644 --- a/build/linux.aarch64/Dockerfile +++ b/build/linux.aarch64/Dockerfile @@ -28,7 +28,7 @@ RUN curl -L "https://downloads.haskell.org/~cabal/cabal-install-3.9.0.0/cabal-in # Due to an apparent cabal bug, we specify our options directly to cabal # It won't reuse caches if ghc-options are specified in ~/.cabal/config -ENV CABALOPTS "--ghc-options;-split-sections -optc-Os -optc-Wl,--gc-sections -optc-fPIC;--with-ghc=$TARGET-ghc;--with-hc-pkg=$TARGET-ghc-pkg" +ENV CABALOPTS "--ghc-options;-split-sections -optc-Os -optc-Wl,--gc-sections -optc-fPIC;--with-ghc=$TARGET-ghc;--with-hc-pkg=$TARGET-ghc-pkg;-c;hashable -arch-native" # Prebuild the dependencies RUN cabal update && IFS=';' && cabal install --dependencies-only $CABALOPTS ShellCheck diff --git a/build/linux.aarch64/build b/build/linux.aarch64/build index f8001aa..3ce61ce 100755 --- a/build/linux.aarch64/build +++ b/build/linux.aarch64/build @@ -4,7 +4,6 @@ set -xe tar xzv --strip-components=1 chmod +x striptests && ./striptests mkdir "$TARGETNAME" - cabal update ( IFS=';'; cabal build $CABALOPTS --enable-executable-static ) find . -name shellcheck -type f -exec mv {} "$TARGETNAME/" \; ls -l "$TARGETNAME" diff --git a/build/linux.armv6hf/Dockerfile b/build/linux.armv6hf/Dockerfile index f933dda..b4d4197 100644 --- a/build/linux.armv6hf/Dockerfile +++ b/build/linux.armv6hf/Dockerfile @@ -1,25 +1,7 @@ -# I've again spent days trying to get a working armv6hf compiler going. -# God only knows how many recompilations of GCC, GHC, libraries, and -# ShellCheck itself, has gone into it. -# -# I tried Debian's toolchain. I tried my custom one built according to -# RPi `gcc -v`. I tried GHC9, glibc, musl, registerised vs not, but -# nothing has yielded an armv6hf binary that does not immediately -# segfault on qemu-arm-static or the RPi itself. -# -# I then tried the same but with armv7hf. Same story. -# -# Emulating the entire userspace with balenalib again? Very strange build -# failures where programs would fail to execute with > ~100 arguments. -# -# Finally, creating our own appears to work when using a custom QEmu -# patched to follow execve calls. -# -# PS: $100 bounty for getting a RPi1 compatible static build going -# with cross-compilation, similar to what the aarch64 build does. -# +# This Docker file uses a custom QEmu fork with patches to follow execve +# to build all of ShellCheck emulated. -FROM ubuntu:20.04 +FROM ubuntu:24.04 ENV TARGETNAME linux.armv6hf @@ -27,34 +9,34 @@ ENV TARGETNAME linux.armv6hf USER root ENV DEBIAN_FRONTEND noninteractive RUN apt-get update -RUN apt-get install -y build-essential git ninja-build python3 pkg-config libglib2.0-dev libpixman-1-dev -WORKDIR /build -RUN git clone --depth 1 https://github.com/koalaman/qemu -RUN cd qemu && ./configure --static && cd build && ninja qemu-arm -RUN cp qemu/build/qemu-arm /build/qemu-arm-static +RUN apt-get install -y --no-install-recommends build-essential git ninja-build python3 pkg-config libglib2.0-dev libpixman-1-dev python3-setuptools ca-certificates debootstrap +WORKDIR /qemu +RUN git clone --depth 1 https://github.com/koalaman/qemu . +RUN ./configure --static --disable-werror && cd build && ninja qemu-arm ENV QEMU_EXECVE 1 +# Convenience utility +COPY scutil /bin/scutil +COPY scutil /chroot/bin/scutil +RUN chmod +x /bin/scutil /chroot/bin/scutil + # Set up an armv6 userspace WORKDIR / -RUN apt-get install -y debootstrap qemu-user-static -# We expect this to fail if the host doesn't have binfmt qemu support -RUN qemu-debootstrap --arch armhf bullseye pi http://mirrordirector.raspbian.org/raspbian || [ -e /pi/etc/issue ] -RUN cp /build/qemu-arm-static /pi/usr/bin/qemu-arm-static -RUN printf > /bin/pirun '%s\n' '#!/bin/sh' 'chroot /pi /usr/bin/qemu-arm-static /usr/bin/env "$@"' && chmod +x /bin/pirun -# If the debootstrap process didn't finish, continue it -RUN [ ! -e /pi/debootstrap ] || pirun '/debootstrap/debootstrap' --second-stage +RUN debootstrap --arch armhf --variant=minbase --foreign bookworm /chroot http://mirrordirector.raspbian.org/raspbian +RUN cp /qemu/build/qemu-arm /chroot/bin/qemu +RUN scutil emu /debootstrap/debootstrap --second-stage # Install deps in the chroot -RUN pirun apt-get update -RUN pirun apt-get install -y ghc cabal-install +RUN scutil emu apt-get update +RUN scutil emu apt-get install -y --no-install-recommends ghc cabal-install +RUN scutil emu cabal update # Finally we can build the current dependencies. This takes hours. ENV CABALOPTS "--ghc-options;-split-sections -optc-Os -optc-Wl,--gc-sections;--gcc-options;-Os -Wl,--gc-sections -ffunction-sections -fdata-sections" -RUN pirun cabal update -RUN IFS=";" && pirun cabal install --dependencies-only $CABALOPTS ShellCheck -RUN IFS=';' && pirun cabal install $CABALOPTS --lib fgl +# Generated with `cabal freeze --constraint 'hashable -arch-native'` +COPY cabal.project.freeze /chroot/etc +RUN IFS=";" && scutil install_from_freeze /chroot/etc/cabal.project.freeze emu cabal install $CABALOPTS # Copy the build script -WORKDIR /pi/scratch -COPY build /pi/usr/bin -ENTRYPOINT ["/bin/pirun", "/usr/bin/build"] +COPY build /chroot/bin +ENTRYPOINT ["/bin/scutil", "emu", "/bin/build"] diff --git a/build/linux.armv6hf/build b/build/linux.armv6hf/build index daa94d9..1d496ae 100755 --- a/build/linux.armv6hf/build +++ b/build/linux.armv6hf/build @@ -1,8 +1,9 @@ #!/bin/sh set -xe -cd /scratch +mkdir /scratch && cd /scratch { tar xzv --strip-components=1 + cp /etc/cabal.project.freeze . chmod +x striptests && ./striptests mkdir "$TARGETNAME" # This script does not cabal update because compiling anything new is slow diff --git a/build/linux.armv6hf/cabal.project.freeze b/build/linux.armv6hf/cabal.project.freeze new file mode 100644 index 0000000..183bcc6 --- /dev/null +++ b/build/linux.armv6hf/cabal.project.freeze @@ -0,0 +1,93 @@ +active-repositories: hackage.haskell.org:merge +constraints: any.Diff ==0.5, + any.OneTuple ==0.4.2, + any.QuickCheck ==2.14.3, + QuickCheck -old-random +templatehaskell, + any.StateVar ==1.2.2, + any.aeson ==2.2.3.0, + aeson +ordered-keymap, + any.array ==0.5.4.0, + any.assoc ==1.1.1, + assoc -tagged, + any.base ==4.15.1.0, + any.base-orphans ==0.9.2, + any.bifunctors ==5.6.2, + bifunctors +tagged, + any.binary ==0.8.8.0, + any.bytestring ==0.10.12.1, + any.character-ps ==0.1, + any.comonad ==5.0.8, + comonad +containers +distributive +indexed-traversable, + any.containers ==0.6.4.1, + any.contravariant ==1.5.5, + contravariant +semigroups +statevar +tagged, + any.data-array-byte ==0.1.0.1, + any.data-fix ==0.3.3, + any.deepseq ==1.4.5.0, + any.directory ==1.3.6.2, + any.distributive ==0.6.2.1, + distributive +semigroups +tagged, + any.dlist ==1.0, + dlist -werror, + any.exceptions ==0.10.4, + any.fgl ==5.8.2.0, + fgl +containers042, + any.filepath ==1.4.2.1, + any.foldable1-classes-compat ==0.1, + foldable1-classes-compat +tagged, + any.generically ==0.1.1, + any.ghc-bignum ==1.1, + any.ghc-boot-th ==9.0.2, + any.ghc-prim ==0.7.0, + any.hashable ==1.4.6.0, + hashable -arch-native +integer-gmp -random-initial-seed, + any.indexed-traversable ==0.1.4, + any.indexed-traversable-instances ==0.1.2, + any.integer-conversion ==0.1.1, + any.integer-logarithms ==1.0.3.1, + integer-logarithms -check-bounds +integer-gmp, + any.mtl ==2.2.2, + any.network-uri ==2.6.4.2, + any.parsec ==3.1.14.0, + any.pretty ==1.1.3.6, + any.primitive ==0.9.0.0, + any.process ==1.6.13.2, + any.random ==1.2.1.2, + any.regex-base ==0.94.0.2, + any.regex-tdfa ==1.3.2.2, + regex-tdfa +doctest -force-o2, + any.rts ==1.0.2, + any.scientific ==0.3.8.0, + scientific -integer-simple, + any.semialign ==1.3.1, + semialign +semigroupoids, + any.semigroupoids ==6.0.1, + semigroupoids +comonad +containers +contravariant +distributive +tagged +unordered-containers, + any.splitmix ==0.1.0.5, + splitmix -optimised-mixer, + any.stm ==2.5.0.0, + any.strict ==0.5, + any.tagged ==0.8.8, + tagged +deepseq +transformers, + any.template-haskell ==2.17.0.0, + any.text ==1.2.5.0, + any.text-iso8601 ==0.1.1, + any.text-short ==0.1.6, + text-short -asserts, + any.th-abstraction ==0.7.0.0, + any.th-compat ==0.1.5, + any.these ==1.2.1, + any.time ==1.9.3, + any.time-compat ==1.9.7, + any.transformers ==0.5.6.2, + any.transformers-compat ==0.7.2, + transformers-compat -five +five-three -four +generic-deriving +mtl -three -two, + any.unix ==2.7.2.2, + any.unordered-containers ==0.2.20, + unordered-containers -debug, + any.uuid-types ==1.0.6, + any.vector ==0.13.1.0, + vector +boundschecks -internalchecks -unsafechecks -wall, + any.vector-stream ==0.1.0.1, + any.witherable ==0.5 +index-state: hackage.haskell.org 2024-06-18T02:21:19Z diff --git a/build/linux.armv6hf/scutil b/build/linux.armv6hf/scutil new file mode 100644 index 0000000..a85d810 --- /dev/null +++ b/build/linux.armv6hf/scutil @@ -0,0 +1,48 @@ +#!/bin/dash +# Various ShellCheck build utility functions + +# Generally set a ulimit to avoid QEmu using too much memory +ulimit -v "$((10*1024*1024))" +# If we happen to invoke or run under QEmu, make sure to follow execve. +# This requires a patched QEmu. +export QEMU_EXECVE=1 + +# Retry a command until it succeeds +# Usage: scutil retry 3 mycmd +retry() { + n="$1" + ret=1 + shift + while [ "$n" -gt 0 ] + do + "$@" + ret=$? + [ "$ret" = 0 ] && break + n=$((n-1)) + done + return "$ret" +} + +# Install all dependencies from a freeze file +# Usage: scutil install_from_freeze /path/cabal.project.freeze cabal install +install_from_freeze() { + linefeed=$(printf '\nx') + linefeed=${linefeed%x} + flags=$( + sed 's/constraints:/&\n /' "$1" | + grep -vw -e rts -e base | + sed -n -e 's/^ *\([^,]*\).*/\1/p' | + sed -e 's/any\.\([^ ]*\) ==\(.*\)/\1-\2/; te; s/.*/--constraint\n&/; :e') + shift + # shellcheck disable=SC2086 + ( IFS=$linefeed; set -x; "$@" $flags ) +} + +# Run a command under emulation. +# This assumes the correct emulator is named 'qemu' and the chroot is /chroot +# Usage: scutil emu echo "Hello World" +emu() { + chroot /chroot /bin/qemu /usr/bin/env "$@" +} + +"$@" diff --git a/build/linux.riscv64/Dockerfile b/build/linux.riscv64/Dockerfile index 0ac95e5..d138ff7 100644 --- a/build/linux.riscv64/Dockerfile +++ b/build/linux.riscv64/Dockerfile @@ -10,46 +10,37 @@ ENV DEBIAN_FRONTEND noninteractive RUN apt-get update -y # Install qemu -RUN apt-get install -y --no-install-recommends build-essential ninja-build python3 pkg-config libglib2.0-dev libpixman-1-dev curl ca-certificates python3-virtualenv git python3-setuptools +RUN apt-get install -y --no-install-recommends build-essential ninja-build python3 pkg-config libglib2.0-dev libpixman-1-dev curl ca-certificates python3-virtualenv git python3-setuptools debootstrap WORKDIR /qemu RUN git clone --depth 1 https://github.com/koalaman/qemu . -#RUN git clone https://github.com/balena-io/qemu . -# Release 7.0.0 -#RUN git checkout 639d1d8903f65d74eb04c49e0df7a4b2f014cd86 RUN ./configure --target-list=riscv64-linux-user --static --disable-system --disable-pie --disable-werror RUN cd build && ninja qemu-riscv64 ENV QEMU_EXECVE 1 +# Convenience utility +COPY scutil /bin/scutil +# We have to copy to /usr/bin because debootstrap will try to symlink /bin and fail if it exists +COPY scutil /chroot/usr/bin/scutil +RUN chmod +x /bin/scutil /chroot/usr/bin/scutil # Set up a riscv64 userspace -RUN apt-get install -y --no-install-recommends debootstrap -RUN debootstrap --arch=riscv64 --foreign noble /rvfs http://ports.ubuntu.com/ubuntu-ports -RUN cp /qemu/build/qemu-riscv64 /rvfs/usr/bin/qemu-riscv64-static - -# Command to run riscv binaries in the chroot. The Haskell runtime allocates 1TB -# vspace up front and QEmu has a RAM cost per vspace, so use ulimit to allocate -# less and reduce RAM usage. -RUN printf > /bin/rv '%s\n' '#!/bin/sh' 'ulimit -v $((10*1024*1024)); chroot /rvfs /usr/bin/qemu-riscv64-static /usr/bin/env "$@"' -RUN chmod +x /bin/rv -RUN [ ! -e /rvfs/debootstrap ] || rv '/debootstrap/debootstrap' --second-stage +WORKDIR / +RUN debootstrap --arch=riscv64 --variant=minbase --components=main,universe --foreign noble /chroot http://ports.ubuntu.com/ubuntu-ports +RUN cp /qemu/build/qemu-riscv64 /chroot/bin/qemu +RUN scutil emu /debootstrap/debootstrap --second-stage # Install deps in the chroot -RUN printf > /rvfs/etc/apt/sources.list '%s\n' 'deb http://ports.ubuntu.com/ubuntu-ports noble main universe' -RUN rv apt-get update -y -RUN rv apt-get install -y --no-install-recommends ghc cabal-install +RUN scutil emu apt-get update +RUN scutil emu apt-get install -y --no-install-recommends ghc cabal-install +RUN scutil emu cabal update -RUN rv cabal update # Generated with: cabal freeze -c 'hashable -arch-native'. We put it in /etc so cabal won't find it. -COPY cabal.project.freeze /rvfs/etc -# Awful hack to install everything from the freeze file -# This basically turns 'any.tagged ==0.8.8' into tagged-0.8.8 to install by version, -# and adds a -c before 'hashable -arch-native +integer-gmp' to make it a flag constraint. -RUN < /rvfs/etc/cabal.project.freeze sed 's/constraints:/&\n /' | grep -vw rts | sed -n -e 's/^ *\([^,]*\).*/\1/p' | sed -e 's/any\.\([^ ]*\) ==\(.*\)/\1-\2/; te; s/.*/-c\n&/; :e' > /tmp/preinstall-flags -# Finally we can build the current dependencies. This takes hours. -# There's apparently a random segfault during assembly, so retry a few times. -RUN set -x; IFS=${IFS# }; f() { rv cabal install --keep-going $(cat /tmp/preinstall-flags); }; for i in $(seq 5); do f; ret=$?; [ $ret = 0 ] && break; done; exit $ret +COPY cabal.project.freeze /chroot/etc + +# Build all dependencies from the freeze file. The emulator segfaults at random, +# so retry a few times. +RUN scutil install_from_freeze /chroot/etc/cabal.project.freeze retry 5 emu cabal install --keep-going # Copy the build script -WORKDIR /rvfs/scratch -COPY build /rvfs/usr/bin/build -ENTRYPOINT ["/bin/rv", "/usr/bin/build"] +COPY build /chroot/bin/build +ENTRYPOINT ["/bin/scutil", "emu", "/bin/build"] diff --git a/build/linux.riscv64/build b/build/linux.riscv64/build index 63c487f..ed9dc27 100755 --- a/build/linux.riscv64/build +++ b/build/linux.riscv64/build @@ -2,6 +2,8 @@ set -xe IFS=';' { + mkdir -p /tmp/scratch + cd /tmp/scratch tar xzv --strip-components=1 chmod +x striptests && ./striptests # Use a freeze file to ensure we use the same dependencies we cached during @@ -9,7 +11,7 @@ IFS=';' cp /etc/cabal.project.freeze . mkdir "$TARGETNAME" # Retry in case of random segfault - cabal build --enable-executable-static || cabal build --enable-executable-static + scutil retry 3 cabal build --enable-executable-static find . -name shellcheck -type f -exec mv {} "$TARGETNAME/" \; ls -l "$TARGETNAME" "$TARGET-strip" -s "$TARGETNAME/shellcheck" diff --git a/build/windows.x86_64/build b/build/windows.x86_64/build index 7bf186e..22e5b42 100755 --- a/build/windows.x86_64/build +++ b/build/windows.x86_64/build @@ -8,7 +8,6 @@ set -xe tar xzv --strip-components=1 chmod +x striptests && ./striptests mkdir "$TARGETNAME" - cabal update ( IFS=';'; cabal build $CABALOPTS ) find dist*/ -name shellcheck.exe -type f -ls -exec mv {} "$TARGETNAME/" \; ls -l "$TARGETNAME" From b408f546209bdd344177d2d778b38d3a77f0db78 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 7 Jul 2024 01:03:40 -0400 Subject: [PATCH 712/763] Simplify invokedNodes --- src/ShellCheck/CFGAnalysis.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/CFGAnalysis.hs b/src/ShellCheck/CFGAnalysis.hs index 0b99c9f..8534d6f 100644 --- a/src/ShellCheck/CFGAnalysis.hs +++ b/src/ShellCheck/CFGAnalysis.hs @@ -1350,7 +1350,7 @@ analyzeControlFlow params t = -- All nodes we've touched invocations <- readSTRef $ cInvocations ctx - let invokedNodes = M.fromDistinctAscList $ map (\c -> (c, ())) $ S.toList $ M.keysSet $ groupByNode $ M.map snd invocations + let invokedNodes = M.fromSet (const ()) $ S.unions $ map (M.keysSet . snd) $ M.elems invocations -- Invoke all functions that were declared but not invoked -- This is so that we still get warnings for dead code From 61b7e66f809d9c2d1be06e5787ae6a98039e798e Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 7 Jul 2024 01:07:39 -0400 Subject: [PATCH 713/763] Use sets instead of maps that never use their values --- src/ShellCheck/Analytics.hs | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index a89f940..621f70a 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -491,14 +491,14 @@ checkWrongArithmeticAssignment params (T_SimpleCommand id [T_Assignment _ _ _ _ sequence_ $ do str <- getNormalString val var:op:_ <- matchRegex regex str - Map.lookup var references + guard $ S.member var references return . warn (getId val) 2100 $ "Use $((..)) for arithmetics, e.g. i=$((i " ++ op ++ " 2))" where regex = mkRegex "^([_a-zA-Z][_a-zA-Z0-9]*)([+*-]).+$" - references = foldl (flip ($)) Map.empty (map insertRef $ variableFlow params) + references = foldl (flip ($)) S.empty (map insertRef $ variableFlow params) insertRef (Assignment (_, _, name, _)) = - Map.insert name () + S.insert name insertRef _ = Prelude.id getNormalString (T_NormalWord _ words) = do @@ -974,32 +974,32 @@ prop_checkArrayWithoutIndex9 = verifyTree checkArrayWithoutIndex "read -r -a arr prop_checkArrayWithoutIndex10 = verifyTree checkArrayWithoutIndex "read -ra arr <<< 'foo bar'; echo \"$arr\"" prop_checkArrayWithoutIndex11 = verifyNotTree checkArrayWithoutIndex "read -rpfoobar r; r=42" checkArrayWithoutIndex params _ = - doVariableFlowAnalysis readF writeF defaultMap (variableFlow params) + doVariableFlowAnalysis readF writeF defaultSet (variableFlow params) where - defaultMap = Map.fromList $ map (\x -> (x,())) arrayVariables + defaultSet = S.fromList arrayVariables readF _ (T_DollarBraced id _ token) _ = do - map <- get + s <- get return . maybeToList $ do name <- getLiteralString token - assigned <- Map.lookup name map + guard $ S.member name s return $ makeComment WarningC id 2128 "Expanding an array without an index only gives the first element." readF _ _ _ = return [] writeF _ (T_Assignment id mode name [] _) _ (DataString _) = do - isArray <- gets (Map.member name) + isArray <- gets (S.member name) return $ if not isArray then [] else case mode of Assign -> [makeComment WarningC id 2178 "Variable was used as an array but is now assigned a string."] Append -> [makeComment WarningC id 2179 "Use array+=(\"item\") to append items to an array."] writeF _ t name (DataArray _) = do - modify (Map.insert name ()) + modify (S.insert name) return [] writeF _ expr name _ = do if isIndexed expr - then modify (Map.insert name ()) - else modify (Map.delete name) + then modify (S.insert name) + else modify (S.delete name) return [] isIndexed expr = @@ -3968,12 +3968,12 @@ prop_checkTranslatedStringVariable4 = verifyNot checkTranslatedStringVariable "v prop_checkTranslatedStringVariable5 = verifyNot checkTranslatedStringVariable "foo=var; bar=val2; $\"foo bar\"" checkTranslatedStringVariable params (T_DollarDoubleQuoted id [T_Literal _ s]) | all isVariableChar s - && Map.member s assignments + && S.member s assignments = warnWithFix id 2256 "This translated string is the name of a variable. Flip leading $ and \" if this should be a quoted substitution." (fix id) where - assignments = foldl (flip ($)) Map.empty (map insertAssignment $ variableFlow params) - insertAssignment (Assignment (_, token, name, _)) | isVariableName name = - Map.insert name token + assignments = foldl (flip ($)) S.empty (map insertAssignment $ variableFlow params) + insertAssignment (Assignment (_, _, name, _)) | isVariableName name = + S.insert name insertAssignment _ = Prelude.id fix id = fixWith [replaceStart id params 2 "\"$"] checkTranslatedStringVariable _ _ = return () From 8746c6e7f20fdaa2463ae008dc527375ab76b04e Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 7 Jul 2024 01:07:53 -0400 Subject: [PATCH 714/763] Switch the order of the maps to avoid unnecessary unionWith instead of union --- src/ShellCheck/CFGAnalysis.hs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ShellCheck/CFGAnalysis.hs b/src/ShellCheck/CFGAnalysis.hs index 8534d6f..27098b1 100644 --- a/src/ShellCheck/CFGAnalysis.hs +++ b/src/ShellCheck/CFGAnalysis.hs @@ -133,7 +133,7 @@ internalToExternal s = literalValue = Nothing } } - flatVars = M.unionsWith (\_ last -> last) $ map mapStorage [sGlobalValues s, sLocalValues s, sPrefixValues s] + flatVars = M.unions $ map mapStorage [sPrefixValues s, sLocalValues s, sGlobalValues s] -- Conveniently get the state before a token id getIncomingState :: CFGAnalysis -> Id -> Maybe ProgramState @@ -672,7 +672,7 @@ vmPatch base diff = _ | vmIsQuickEqual base diff -> diff _ -> VersionedMap { mapVersion = -1, - mapStorage = M.unionWith (flip const) (mapStorage base) (mapStorage diff) + mapStorage = M.union (mapStorage diff) (mapStorage base) } -- Set a variable. This includes properties. Applies it to the appropriate scope. @@ -1373,7 +1373,7 @@ analyzeControlFlow params t = -- Fill in the map with unreachable states for anything we didn't get to let baseStates = M.fromDistinctAscList $ map (\c -> (c, (unreachableState, unreachableState))) $ uncurry enumFromTo $ nodeRange $ cfGraph cfg - let allStates = M.unionWith (flip const) baseStates invokedStates + let allStates = M.union invokedStates baseStates -- Convert to external states let nodeToData = M.map (\(a,b) -> (internalToExternal a, internalToExternal b)) allStates From e5fdec970ae3bc16369b3ec289a4c4fbb9254751 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 7 Jul 2024 01:08:18 -0400 Subject: [PATCH 715/763] Swap the order of the tuple returned by orderEdge --- src/ShellCheck/CFG.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/CFG.hs b/src/ShellCheck/CFG.hs index e1d3259..a720911 100644 --- a/src/ShellCheck/CFG.hs +++ b/src/ShellCheck/CFG.hs @@ -300,13 +300,13 @@ removeUnnecessaryStructuralNodes (nodes, edges, mapping, association) = edgesToCollapse = S.fromList $ filter filterEdges regularEdges remapping :: M.Map Node Node - remapping = foldl' (\m (new, old) -> M.insert old new m) M.empty $ map orderEdge $ S.toList edgesToCollapse + remapping = foldl' (\m (old, new) -> M.insert old new m) M.empty $ map orderEdge $ S.toList edgesToCollapse recursiveRemapping = M.fromList $ map (\c -> (c, recursiveLookup remapping c)) $ M.keys remapping filterEdges (a,b,_) = a `S.member` candidateNodes && b `S.member` candidateNodes - orderEdge (a,b,_) = if a < b then (a,b) else (b,a) + orderEdge (a,b,_) = if a < b then (b,a) else (a,b) counter = foldl' (\map key -> M.insertWith (+) key 1 map) M.empty isRegularEdge (_, _, CFEFlow) = True isRegularEdge _ = False From 95c0cc2e4bdbad193ba3616a5ecfd2741b1f3807 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 7 Jul 2024 01:09:17 -0400 Subject: [PATCH 716/763] Simplify removeUnnecessaryStructuralNodes --- src/ShellCheck/CFG.hs | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/src/ShellCheck/CFG.hs b/src/ShellCheck/CFG.hs index a720911..dc56b58 100644 --- a/src/ShellCheck/CFG.hs +++ b/src/ShellCheck/CFG.hs @@ -295,19 +295,19 @@ removeUnnecessaryStructuralNodes (nodes, edges, mapping, association) = regularEdges = filter isRegularEdge edges inDegree = counter $ map (\(from,to,_) -> from) regularEdges outDegree = counter $ map (\(from,to,_) -> to) regularEdges - structuralNodes = S.fromList $ map fst $ filter isStructural nodes + structuralNodes = S.fromList [node | (node, CFStructuralNode) <- nodes] candidateNodes = S.filter isLinear structuralNodes edgesToCollapse = S.fromList $ filter filterEdges regularEdges remapping :: M.Map Node Node - remapping = foldl' (\m (old, new) -> M.insert old new m) M.empty $ map orderEdge $ S.toList edgesToCollapse - recursiveRemapping = M.fromList $ map (\c -> (c, recursiveLookup remapping c)) $ M.keys remapping + remapping = M.fromList $ map orderEdge $ S.toList edgesToCollapse + recursiveRemapping = M.mapWithKey (\c _ -> recursiveLookup remapping c) remapping filterEdges (a,b,_) = a `S.member` candidateNodes && b `S.member` candidateNodes orderEdge (a,b,_) = if a < b then (b,a) else (a,b) - counter = foldl' (\map key -> M.insertWith (+) key 1 map) M.empty + counter = M.fromListWith (+) . map (\key -> (key, 1)) isRegularEdge (_, _, CFEFlow) = True isRegularEdge _ = False @@ -317,11 +317,6 @@ removeUnnecessaryStructuralNodes (nodes, edges, mapping, association) = Nothing -> node Just x -> recursiveLookup map x - isStructural (node, label) = - case label of - CFStructuralNode -> True - _ -> False - isLinear node = M.findWithDefault 0 node inDegree == 1 && M.findWithDefault 0 node outDegree == 1 From 98b8dc0720148d69ff924f89966c50dc2dda2fe3 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 7 Jul 2024 01:11:00 -0400 Subject: [PATCH 717/763] Use fromList instead of reimplementing it in terms of foldl --- src/ShellCheck/Analytics.hs | 20 ++++---------------- src/ShellCheck/CFGAnalysis.hs | 2 +- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 621f70a..211993c 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -496,10 +496,7 @@ checkWrongArithmeticAssignment params (T_SimpleCommand id [T_Assignment _ _ _ _ "Use $((..)) for arithmetics, e.g. i=$((i " ++ op ++ " 2))" where regex = mkRegex "^([_a-zA-Z][_a-zA-Z0-9]*)([+*-]).+$" - references = foldl (flip ($)) S.empty (map insertRef $ variableFlow params) - insertRef (Assignment (_, _, name, _)) = - S.insert name - insertRef _ = Prelude.id + references = S.fromList [name | Assignment (_, _, name, _) <- variableFlow params] getNormalString (T_NormalWord _ words) = do parts <- mapM getLiterals words @@ -2380,15 +2377,9 @@ prop_checkUnused51 = verifyTree checkUnusedAssignments "x[y[z=1]]=1; echo ${x[@] checkUnusedAssignments params t = execWriter (mapM_ warnFor unused) where flow = variableFlow params - references = foldl (flip ($)) defaultMap (map insertRef flow) - insertRef (Reference (base, token, name)) = - Map.insert (stripSuffix name) () - insertRef _ = id + references = Map.union (Map.fromList [(stripSuffix name, ()) | Reference (base, token, name) <- flow]) defaultMap - assignments = foldl (flip ($)) Map.empty (map insertAssignment flow) - insertAssignment (Assignment (_, token, name, _)) | isVariableName name = - Map.insert name token - insertAssignment _ = id + assignments = Map.fromList [(name, token) | Assignment (_, token, name, _) <- flow, isVariableName name] unused = Map.assocs $ Map.difference assignments references @@ -3971,10 +3962,7 @@ checkTranslatedStringVariable params (T_DollarDoubleQuoted id [T_Literal _ s]) && S.member s assignments = warnWithFix id 2256 "This translated string is the name of a variable. Flip leading $ and \" if this should be a quoted substitution." (fix id) where - assignments = foldl (flip ($)) S.empty (map insertAssignment $ variableFlow params) - insertAssignment (Assignment (_, _, name, _)) | isVariableName name = - S.insert name - insertAssignment _ = Prelude.id + assignments = S.fromList [name | Assignment (_, _, name, _) <- variableFlow params, isVariableName name] fix id = fixWith [replaceStart id params 2 "\"$"] checkTranslatedStringVariable _ _ = return () diff --git a/src/ShellCheck/CFGAnalysis.hs b/src/ShellCheck/CFGAnalysis.hs index 27098b1..cf982e0 100644 --- a/src/ShellCheck/CFGAnalysis.hs +++ b/src/ShellCheck/CFGAnalysis.hs @@ -1286,7 +1286,7 @@ dataflow ctx entry = do else do let (next, rest) = S.deleteFindMin ps nexts <- process states next - writeSTRef pending $ foldl (flip S.insert) rest nexts + writeSTRef pending $ S.union (S.fromList nexts) rest f (n-1) pending states process states node = do From 6593096ba06e6b54ec08d00a9c625930a357cdfe Mon Sep 17 00:00:00 2001 From: Sertonix Date: Fri, 28 Jun 2024 16:24:06 +0200 Subject: [PATCH 718/763] Allow SC3003 on busybox shell --- src/ShellCheck/Checks/ShellSupport.hs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index cab0546..9192c0e 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -212,6 +212,8 @@ prop_checkBashisms118 = verify checkBashisms "#!/bin/busybox sh\nxyz=1\n${!x*}" prop_checkBashisms119 = verify checkBashisms "#!/bin/busybox sh\nx='test'\n${x^^[t]}" -- SC3059 prop_checkBashisms120 = verify checkBashisms "#!/bin/sh\n[ x == y ]" prop_checkBashisms121 = verifyNot checkBashisms "#!/bin/sh\n# shellcheck shell=busybox\n[ x == y ]" +prop_checkBashisms122 = verify checkBashisms "#!/bin/dash\n$'a'" +prop_checkBashisms123 = verifyNot checkBashisms "#!/bin/busybox sh\n$'a'" checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do params <- ask kludge params t @@ -229,7 +231,8 @@ checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do bashism (T_ProcSub id _ _) = warnMsg id 3001 "process substitution is" bashism (T_Extglob id _ _) = warnMsg id 3002 "extglob is" - bashism (T_DollarSingleQuoted id _) = warnMsg id 3003 "$'..' is" + bashism (T_DollarSingleQuoted id _) = + unless isBusyboxSh $ warnMsg id 3003 "$'..' is" bashism (T_DollarDoubleQuoted id _) = warnMsg id 3004 "$\"..\" is" bashism (T_ForArithmetic id _ _ _ _) = warnMsg id 3005 "arithmetic for loops are" bashism (T_Arithmetic id _) = warnMsg id 3006 "standalone ((..)) is" From 4c852749214e102f7a1daec08094ed91b08867f7 Mon Sep 17 00:00:00 2001 From: Sertonix Date: Tue, 2 Jul 2024 17:06:35 +0200 Subject: [PATCH 719/763] Fix SC3045 for busybox shell --- src/ShellCheck/Checks/ShellSupport.hs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 9192c0e..1207ad0 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -214,6 +214,9 @@ prop_checkBashisms120 = verify checkBashisms "#!/bin/sh\n[ x == y ]" prop_checkBashisms121 = verifyNot checkBashisms "#!/bin/sh\n# shellcheck shell=busybox\n[ x == y ]" prop_checkBashisms122 = verify checkBashisms "#!/bin/dash\n$'a'" prop_checkBashisms123 = verifyNot checkBashisms "#!/bin/busybox sh\n$'a'" +prop_checkBashisms124 = verify checkBashisms "#!/bin/dash\ntype -p test" +prop_checkBashisms125 = verifyNot checkBashisms "#!/bin/busybox sh\ntype -p test" +prop_checkBashisms126 = verifyNot checkBashisms "#!/bin/busybox sh\nread -p foo -r bar" checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do params <- ask kludge params t @@ -446,10 +449,10 @@ checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do ("hash", Just $ if isDash then ["r", "v"] else ["r"]), ("jobs", Just ["l", "p"]), ("printf", Just []), - ("read", Just $ if isDash then ["r", "p"] else ["r"]), + ("read", Just $ if isDash || isBusyboxSh then ["r", "p"] else ["r"]), ("readonly", Just ["p"]), ("trap", Just []), - ("type", Just []), + ("type", Just $ if isBusyboxSh then ["p"] else []), ("ulimit", if isDash then Nothing else Just ["f"]), ("umask", Just ["S"]), ("unset", Just ["f", "v"]), From 6d2f3d8628235dd2146cd248d9828b004852c7ad Mon Sep 17 00:00:00 2001 From: Sertonix Date: Tue, 9 Jul 2024 15:12:16 +0200 Subject: [PATCH 720/763] Allow 'echo -e' in busybox shell --- src/ShellCheck/Checks/ShellSupport.hs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 1207ad0..2ec4cf7 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -217,6 +217,7 @@ prop_checkBashisms123 = verifyNot checkBashisms "#!/bin/busybox sh\n$'a'" prop_checkBashisms124 = verify checkBashisms "#!/bin/dash\ntype -p test" prop_checkBashisms125 = verifyNot checkBashisms "#!/bin/busybox sh\ntype -p test" prop_checkBashisms126 = verifyNot checkBashisms "#!/bin/busybox sh\nread -p foo -r bar" +prop_checkBashisms127 = verifyNot checkBashisms "#!/bin/busybox sh\necho -ne foo" checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do params <- ask kludge params t @@ -327,7 +328,11 @@ checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do bashism t@(T_SimpleCommand _ _ (cmd:arg:_)) | t `isCommand` "echo" && argString `matches` flagRegex = - if isDash + if isBusyboxSh + then + when (not (argString `matches` busyboxFlagRegex)) $ + warnMsg (getId arg) 3036 "echo flags besides -n and -e" + else if isDash then when (argString /= "-n") $ warnMsg (getId arg) 3036 "echo flags besides -n" @@ -336,6 +341,7 @@ checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do where argString = concat $ oversimplify arg flagRegex = mkRegex "^-[eEsn]+$" + busyboxFlagRegex = mkRegex "^-[en]+$" bashism t@(T_SimpleCommand _ _ (cmd:arg:_)) | getLiteralString cmd == Just "exec" && "-" `isPrefixOf` concat (oversimplify arg) = From d590a35ff8093a3a1edeac67cdb7fb9cffa593bf Mon Sep 17 00:00:00 2001 From: Hasit Mistry Date: Tue, 9 Jul 2024 14:22:19 -0700 Subject: [PATCH 721/763] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 366c427..03d4d4a 100644 --- a/README.md +++ b/README.md @@ -113,6 +113,7 @@ Services and platforms that have ShellCheck pre-installed and ready to use: * [CircleCI](https://circleci.com) via the [ShellCheck Orb](https://circleci.com/orbs/registry/orb/circleci/shellcheck) * [Github](https://github.com/features/actions) (only Linux) * [Trunk Check](https://trunk.io/products/check) (universal linter; [allows you to explicitly version your shellcheck install](https://github.com/trunk-io/plugins/blob/bcbb361dcdbe4619af51ea7db474d7fb87540d20/.trunk/trunk.yaml#L32)) via the [shellcheck plugin](https://github.com/trunk-io/plugins/blob/main/linters/shellcheck/plugin.yaml) +* [CodeRabbit](https://coderabbit.ai/) Most other services, including [GitLab](https://about.gitlab.com/), let you install ShellCheck yourself, either through the system's package manager (see [Installing](#installing)), From 2696c6472dd55880f4dc1350262e1192953754ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Santoro?= Date: Wed, 31 Jul 2024 12:52:42 +0000 Subject: [PATCH 722/763] Whitelist oc to avoid SC2016 false positive Fixes #3033. --- src/ShellCheck/Analytics.hs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 211993c..08eed57 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1097,6 +1097,7 @@ checkSingleQuotedVariables params t@(T_SingleQuoted id s) = ,"sudo" -- covering "sudo sh" and such ,"docker" -- like above ,"podman" + ,"oc" ,"dpkg-query" ,"jq" -- could also check that user provides --arg ,"rename" From 38c5ba7c79e35af29bb1496e774af1d5add0e73c Mon Sep 17 00:00:00 2001 From: Emil Berg Date: Sat, 3 Aug 2024 08:49:40 +0200 Subject: [PATCH 723/763] Fix typos and trailing whitespace --- .github/ISSUE_TEMPLATE.md | 4 ++-- README.md | 2 +- src/ShellCheck/AST.hs | 4 ++-- src/ShellCheck/Analytics.hs | 6 +++--- src/ShellCheck/CFG.hs | 2 +- src/ShellCheck/Checks/Custom.hs | 2 +- src/ShellCheck/Parser.hs | 2 +- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 44d151e..493b465 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -1,6 +1,6 @@ #### For bugs -- Rule Id (if any, e.g. SC1000): -- My shellcheck version (`shellcheck --version` or "online"): +- Rule Id (if any, e.g. SC1000): +- My shellcheck version (`shellcheck --version` or "online"): - [ ] The rule's wiki page does not already cover this (e.g. https://shellcheck.net/wiki/SC2086) - [ ] I tried on https://www.shellcheck.net/ and verified that this is still a problem on the latest commit diff --git a/README.md b/README.md index 366c427..78802a0 100644 --- a/README.md +++ b/README.md @@ -233,7 +233,7 @@ Alternatively, you can download pre-compiled binaries for the latest release her * [Linux, x86_64](https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.x86_64.tar.xz) (statically linked) * [Linux, armv6hf](https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.armv6hf.tar.xz), i.e. Raspberry Pi (statically linked) * [Linux, aarch64](https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.aarch64.tar.xz) aka ARM64 (statically linked) -* [macOS, aarch64](https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.darwin.aarch64.tar.xz) +* [macOS, aarch64](https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.darwin.aarch64.tar.xz) * [macOS, x86_64](https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.darwin.x86_64.tar.xz) * [Windows, x86](https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.zip) diff --git a/src/ShellCheck/AST.hs b/src/ShellCheck/AST.hs index ca05c98..979a9b0 100644 --- a/src/ShellCheck/AST.hs +++ b/src/ShellCheck/AST.hs @@ -206,7 +206,7 @@ pattern T_Annotation id anns t = OuterToken id (Inner_T_Annotation anns t) pattern T_Arithmetic id c = OuterToken id (Inner_T_Arithmetic c) pattern T_Array id t = OuterToken id (Inner_T_Array t) pattern TA_Sequence id l = OuterToken id (Inner_TA_Sequence l) -pattern TA_Parentesis id t = OuterToken id (Inner_TA_Parenthesis t) +pattern TA_Parenthesis id t = OuterToken id (Inner_TA_Parenthesis t) pattern T_Assignment id mode var indices value = OuterToken id (Inner_T_Assignment mode var indices value) pattern TA_Trinary id t1 t2 t3 = OuterToken id (Inner_TA_Trinary t1 t2 t3) pattern TA_Unary id op t1 = OuterToken id (Inner_TA_Unary op t1) @@ -259,7 +259,7 @@ pattern T_Subshell id l = OuterToken id (Inner_T_Subshell l) pattern T_UntilExpression id c l = OuterToken id (Inner_T_UntilExpression c l) pattern T_WhileExpression id c l = OuterToken id (Inner_T_WhileExpression c l) -{-# COMPLETE T_AND_IF, T_Bang, T_Case, TC_Empty, T_CLOBBER, T_DGREAT, T_DLESS, T_DLESSDASH, T_Do, T_DollarSingleQuoted, T_Done, T_DSEMI, T_Elif, T_Else, T_EOF, T_Esac, T_Fi, T_For, T_Glob, T_GREATAND, T_Greater, T_If, T_In, T_Lbrace, T_Less, T_LESSAND, T_LESSGREAT, T_Literal, T_Lparen, T_NEWLINE, T_OR_IF, T_ParamSubSpecialChar, T_Pipe, T_Rbrace, T_Rparen, T_Select, T_Semi, T_SingleQuoted, T_Then, T_UnparsedIndex, T_Until, T_While, TA_Assignment, TA_Binary, TA_Expansion, T_AndIf, T_Annotation, T_Arithmetic, T_Array, TA_Sequence, TA_Parentesis, T_Assignment, TA_Trinary, TA_Unary, TA_Variable, T_Backgrounded, T_Backticked, T_Banged, T_BatsTest, T_BraceExpansion, T_BraceGroup, TC_And, T_CaseExpression, TC_Binary, TC_Group, TC_Nullary, T_Condition, T_CoProcBody, T_CoProc, TC_Or, TC_Unary, T_DollarArithmetic, T_DollarBraceCommandExpansion, T_DollarBraced, T_DollarBracket, T_DollarDoubleQuoted, T_DollarExpansion, T_DoubleQuoted, T_Extglob, T_FdRedirect, T_ForArithmetic, T_ForIn, T_Function, T_HereDoc, T_HereString, T_IfExpression, T_Include, T_IndexedElement, T_IoDuplicate, T_IoFile, T_NormalWord, T_OrIf, T_Pipeline, T_ProcSub, T_Redirecting, T_Script, T_SelectIn, T_SimpleCommand, T_SourceCommand, T_Subshell, T_UntilExpression, T_WhileExpression #-} +{-# COMPLETE T_AND_IF, T_Bang, T_Case, TC_Empty, T_CLOBBER, T_DGREAT, T_DLESS, T_DLESSDASH, T_Do, T_DollarSingleQuoted, T_Done, T_DSEMI, T_Elif, T_Else, T_EOF, T_Esac, T_Fi, T_For, T_Glob, T_GREATAND, T_Greater, T_If, T_In, T_Lbrace, T_Less, T_LESSAND, T_LESSGREAT, T_Literal, T_Lparen, T_NEWLINE, T_OR_IF, T_ParamSubSpecialChar, T_Pipe, T_Rbrace, T_Rparen, T_Select, T_Semi, T_SingleQuoted, T_Then, T_UnparsedIndex, T_Until, T_While, TA_Assignment, TA_Binary, TA_Expansion, T_AndIf, T_Annotation, T_Arithmetic, T_Array, TA_Sequence, TA_Parenthesis, T_Assignment, TA_Trinary, TA_Unary, TA_Variable, T_Backgrounded, T_Backticked, T_Banged, T_BatsTest, T_BraceExpansion, T_BraceGroup, TC_And, T_CaseExpression, TC_Binary, TC_Group, TC_Nullary, T_Condition, T_CoProcBody, T_CoProc, TC_Or, TC_Unary, T_DollarArithmetic, T_DollarBraceCommandExpansion, T_DollarBraced, T_DollarBracket, T_DollarDoubleQuoted, T_DollarExpansion, T_DoubleQuoted, T_Extglob, T_FdRedirect, T_ForArithmetic, T_ForIn, T_Function, T_HereDoc, T_HereString, T_IfExpression, T_Include, T_IndexedElement, T_IoDuplicate, T_IoFile, T_NormalWord, T_OrIf, T_Pipeline, T_ProcSub, T_Redirecting, T_Script, T_SelectIn, T_SimpleCommand, T_SourceCommand, T_Subshell, T_UntilExpression, T_WhileExpression #-} instance Eq Token where OuterToken _ a == OuterToken _ b = a == b diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 211993c..b3d673f 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -3294,7 +3294,7 @@ checkReturnAgainstZero params token = next@(TA_Unary _ "!" _):_ -> isOnlyTestInCommand next next@(TC_Group {}):_ -> isOnlyTestInCommand next next@(TA_Sequence _ [_]):_ -> isOnlyTestInCommand next - next@(TA_Parentesis _ _):_ -> isOnlyTestInCommand next + next@(TA_Parenthesis _ _):_ -> isOnlyTestInCommand next _ -> False -- TODO: Do better $? tracking and filter on whether @@ -4990,14 +4990,14 @@ checkUnnecessaryParens params t = T_ForArithmetic _ x y z _ -> mapM_ (checkLeading "for (((x); (y); (z))) is the same as for ((x; y; z))") [x,y,z] T_Assignment _ _ _ [t] _ -> checkLeading "a[(x)] is the same as a[x]" t T_Arithmetic _ t -> checkLeading "(( (x) )) is the same as (( x ))" t - TA_Parentesis _ (TA_Sequence _ [ TA_Parentesis id _ ]) -> + TA_Parenthesis _ (TA_Sequence _ [ TA_Parenthesis id _ ]) -> styleWithFix id 2322 "In arithmetic contexts, ((x)) is the same as (x). Prefer only one layer of parentheses." $ fix id _ -> return () where checkLeading str t = case t of - TA_Sequence _ [TA_Parentesis id _ ] -> styleWithFix id 2323 (str ++ ". Prefer not wrapping in additional parentheses.") $ fix id + TA_Sequence _ [TA_Parenthesis id _ ] -> styleWithFix id 2323 (str ++ ". Prefer not wrapping in additional parentheses.") $ fix id _ -> return () fix id = diff --git a/src/ShellCheck/CFG.hs b/src/ShellCheck/CFG.hs index dc56b58..81689a1 100644 --- a/src/ShellCheck/CFG.hs +++ b/src/ShellCheck/CFG.hs @@ -490,7 +490,7 @@ build t = do TA_Binary _ _ a b -> sequentially [a,b] TA_Expansion _ list -> sequentially list TA_Sequence _ list -> sequentially list - TA_Parentesis _ t -> build t + TA_Parenthesis _ t -> build t TA_Trinary _ cond a b -> do condition <- build cond diff --git a/src/ShellCheck/Checks/Custom.hs b/src/ShellCheck/Checks/Custom.hs index 76ac83c..17e9c9e 100644 --- a/src/ShellCheck/Checks/Custom.hs +++ b/src/ShellCheck/Checks/Custom.hs @@ -1,7 +1,7 @@ {- This empty file is provided for ease of patching in site specific checks. However, there are no guarantees regarding compatibility between versions. --} +-} {-# LANGUAGE TemplateHaskell #-} module ShellCheck.Checks.Custom (checker, ShellCheck.Checks.Custom.runTests) where diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 9cc5e02..dfe3131 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -827,7 +827,7 @@ readArithmeticContents = char ')' id <- endSpan start spacing - return $ TA_Parentesis id s + return $ TA_Parenthesis id s readArithTerm = readGroup <|> readVariable <|> readExpansion From c7611dfcc6ccb320b530a4e9179e6facee96a422 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 19 Aug 2024 18:37:29 -0700 Subject: [PATCH 724/763] Use dynamic artifact name to work around issue with v4 uploader --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8f9d7d0..3886655 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -65,7 +65,7 @@ jobs: - name: Upload artifact uses: actions/upload-artifact@v4 with: - name: bin + name: ${{matrix.build}}.bin path: bin/ package_binary: From 68e6f02267defb1e2e6398b0f477b4b85e930c22 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 31 Aug 2024 18:00:49 -0700 Subject: [PATCH 725/763] Expand list of recognized unicode spaces (and rewrite for performance) --- src/ShellCheck/Parser.hs | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index dfe3131..3de3537 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -141,15 +141,9 @@ carriageReturn = do parseProblemAt pos ErrorC 1017 "Literal carriage return. Run script through tr -d '\\r' ." return '\r' -almostSpace = - choice [ - check '\xA0' "unicode non-breaking space", - check '\x200B' "unicode zerowidth space" - ] - where - check c name = do - parseNote ErrorC 1018 $ "This is a " ++ name ++ ". Delete and retype it." - char c +almostSpace = do + parseNote ErrorC 1018 $ "This is a unicode space. Delete and retype it." + oneOf "\xA0\x2002\x2003\x2004\x2005\x2006\x2007\x2008\x2009\x200B\x202F" return ' ' --------- Message/position annotation on top of user state From 1487e57a46a48f926f0cd698756cfe41d9635c15 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 31 Aug 2024 18:27:18 -0700 Subject: [PATCH 726/763] Suppress unused warnings about stderr and stderr_lines from bats tests, fixing tests. --- src/ShellCheck/Data.hs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/ShellCheck/Data.hs b/src/ShellCheck/Data.hs index 917142e..3000a99 100644 --- a/src/ShellCheck/Data.hs +++ b/src/ShellCheck/Data.hs @@ -62,6 +62,9 @@ internalVariables = [ , "FLAGS_ARGC", "FLAGS_ARGV", "FLAGS_ERROR", "FLAGS_FALSE", "FLAGS_HELP", "FLAGS_PARENT", "FLAGS_RESERVED", "FLAGS_TRUE", "FLAGS_VERSION", "flags_error", "flags_return" + + -- Bats + ,"stderr", "stderr_lines" ] specialIntegerVariables = [ From 88e441453ba3bbb2aa0449dc178b8f82aaee5b4c Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 31 Aug 2024 18:31:47 -0700 Subject: [PATCH 727/763] Make SC2002 optional (useless-use-of-cat) --- src/ShellCheck/Analytics.hs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 869b683..ce11884 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -103,8 +103,7 @@ nodeChecksToTreeCheck checkList = nodeChecks :: [Parameters -> Token -> Writer [TokenComment] ()] nodeChecks = [ - checkUuoc - ,checkPipePitfalls + checkPipePitfalls ,checkForInQuoted ,checkForInLs ,checkShorthandIf @@ -273,6 +272,13 @@ optionalTreeChecks = [ cdPositive = "rm -r \"$(get_chroot_dir)/home\"", cdNegative = "set -e; dir=\"$(get_chroot_dir)\"; rm -r \"$dir/home\"" }, checkExtraMaskedReturns) + + ,(newCheckDescription { + cdName = "useless-use-of-cat", + cdDescription = "Check for Useless Use Of Cat (UUOC)", + cdPositive = "cat foo | grep bar", + cdNegative = "grep bar foo" + }, nodeChecksToTreeCheck [checkUuoc]) ] optionalCheckMap :: Map.Map String (Parameters -> Token -> [TokenComment]) From 8a1b24c7afcd0534e15eb7db8e386d0d550c6015 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 1 Sep 2024 13:21:44 -0700 Subject: [PATCH 728/763] Fix paths for CI binary packaging after upgrade --- .github/workflows/build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3886655..b0d1085 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -43,7 +43,7 @@ jobs: path: source/ build_source: - name: Build Source Code + name: Build needs: package_source strategy: matrix: @@ -80,13 +80,13 @@ jobs: uses: actions/download-artifact@v4 - name: Work around GitHub permissions bug - run: chmod +x bin/*/shellcheck* + run: chmod +x *.bin/*/shellcheck* - name: Package binaries run: | export TAGS="$(cat source/tags)" mkdir -p deploy - cp -r bin/* deploy + cp -r *.bin/* deploy cd deploy ../.prepare_deploy rm -rf */ README* LICENSE* From ca65071d778d140bc6bad64e699265dde25821e9 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 1 Sep 2024 14:06:26 -0700 Subject: [PATCH 729/763] Run unit tests in GitHub actions --- .github/workflows/build.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b0d1085..83269c9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -42,6 +42,29 @@ jobs: name: source path: source/ + run_tests: + name: Run tests + needs: package_source + runs-on: ubuntu-latest + steps: + - name: Download artifacts + uses: actions/download-artifact@v4 + + - name: Install dependencies + run: | + sudo apt-get update && sudo apt-get install ghc cabal-install + cabal update + + - name: Unpack source + run: | + cd source + tar xvf source.tar.gz --strip-components=1 + + - name: Build and run tests + run: | + cd source + cabal test + build_source: name: Build needs: package_source From 79e43c4550aaf50ebcced1c2b2852f1bd2533f6c Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 7 Sep 2024 17:14:52 -0700 Subject: [PATCH 730/763] Allow parsing arbitrary coproc names (fixes #3048) --- src/ShellCheck/AST.hs | 2 +- src/ShellCheck/AnalyzerLib.hs | 8 ++++++-- src/ShellCheck/CFG.hs | 14 +++++++++++--- src/ShellCheck/Parser.hs | 35 ++++++++++++++++++++++++++++------- 4 files changed, 46 insertions(+), 13 deletions(-) diff --git a/src/ShellCheck/AST.hs b/src/ShellCheck/AST.hs index 979a9b0..bafe035 100644 --- a/src/ShellCheck/AST.hs +++ b/src/ShellCheck/AST.hs @@ -138,7 +138,7 @@ data InnerToken t = | Inner_T_WhileExpression [t] [t] | Inner_T_Annotation [Annotation] t | Inner_T_Pipe String - | Inner_T_CoProc (Maybe String) t + | Inner_T_CoProc (Maybe Token) t | Inner_T_CoProcBody t | Inner_T_Include t | Inner_T_SourceCommand t t diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 3b1faa9..531ce8b 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -559,8 +559,12 @@ getModifiedVariables t = T_FdRedirect _ ('{':var) op -> -- {foo}>&2 modifies foo [(t, t, takeWhile (/= '}') var, DataString SourceInteger) | not $ isClosingFileOp op] - T_CoProc _ name _ -> - [(t, t, fromMaybe "COPROC" name, DataArray SourceInteger)] + T_CoProc _ Nothing _ -> + [(t, t, "COPROC", DataArray SourceInteger)] + + T_CoProc _ (Just token) _ -> do + name <- maybeToList $ getLiteralString token + [(t, t, name, DataArray SourceInteger)] --Points to 'for' rather than variable T_ForIn id str [] _ -> [(t, t, str, DataString SourceExternal)] diff --git a/src/ShellCheck/CFG.hs b/src/ShellCheck/CFG.hs index 81689a1..57aaf4b 100644 --- a/src/ShellCheck/CFG.hs +++ b/src/ShellCheck/CFG.hs @@ -668,10 +668,18 @@ build t = do status <- newNodeRange $ CFSetExitCode id linkRange cond status - T_CoProc id maybeName t -> do - let name = fromMaybe "COPROC" maybeName + T_CoProc id maybeNameToken t -> do + -- If unspecified, "COPROC". If not a constant string, Nothing. + let maybeName = case maybeNameToken of + Just x -> getLiteralString x + Nothing -> Just "COPROC" + + let parentNode = case maybeName of + Just str -> applySingle $ IdTagged id $ CFWriteVariable str CFValueArray + Nothing -> CFStructuralNode + start <- newStructuralNode - parent <- newNodeRange $ applySingle $ IdTagged id $ CFWriteVariable name CFValueArray + parent <- newNodeRange parentNode child <- subshell id "coproc" $ build t end <- newNodeRange $ CFSetExitCode id diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 3de3537..66d62ff 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -2795,17 +2795,29 @@ readFunctionDefinition = called "function" $ do prop_readCoProc1 = isOk readCoProc "coproc foo { echo bar; }" prop_readCoProc2 = isOk readCoProc "coproc { echo bar; }" prop_readCoProc3 = isOk readCoProc "coproc echo bar" +prop_readCoProc4 = isOk readCoProc "coproc a=b echo bar" +prop_readCoProc5 = isOk readCoProc "coproc 'foo' { echo bar; }" +prop_readCoProc6 = isOk readCoProc "coproc \"foo$$\" { echo bar; }" +prop_readCoProc7 = isOk readCoProc "coproc 'foo' ( echo bar )" +prop_readCoProc8 = isOk readCoProc "coproc \"foo$$\" while true; do true; done" readCoProc = called "coproc" $ do start <- startSpan try $ do string "coproc" - whitespace + spacing1 choice [ try $ readCompoundCoProc start, readSimpleCoProc start ] where readCompoundCoProc start = do - var <- optionMaybe $ - readVariableName `thenSkip` whitespace - body <- readBody readCompoundCommand + notFollowedBy2 readAssignmentWord + (var, body) <- choice [ + try $ do + body <- readBody readCompoundCommand + return (Nothing, body), + try $ do + var <- readNormalWord `thenSkip` spacing + body <- readBody readCompoundCommand + return (Just var, body) + ] id <- endSpan start return $ T_CoProc id var body readSimpleCoProc start = do @@ -3436,13 +3448,22 @@ isOk p s = parsesCleanly p s == Just True -- The string parses with no wa isWarning p s = parsesCleanly p s == Just False -- The string parses with warnings isNotOk p s = parsesCleanly p s == Nothing -- The string does not parse -parsesCleanly parser string = runIdentity $ do +-- If the parser matches the string, return Right [ParseNotes+ParseProblems] +-- If it does not match the string, return Left [ParseProblems] +getParseOutput parser string = runIdentity $ do (res, sys) <- runParser testEnvironment (parser >> eof >> getState) "-" string case (res, sys) of (Right userState, systemState) -> - return $ Just . null $ parseNotes userState ++ parseProblems systemState - (Left _, _) -> return Nothing + return $ Right $ parseNotes userState ++ parseProblems systemState + (Left _, systemState) -> return $ Left $ parseProblems systemState + +-- If the parser matches the string, return Just whether it was clean (without emitting suggestions) +-- Otherwise, Nothing +parsesCleanly parser string = + case getParseOutput parser string of + Right list -> Just $ null list + Left _ -> Nothing parseWithNotes parser = do item <- parser From 5c2be767abff85fd10325a2d8a61a372567e63b8 Mon Sep 17 00:00:00 2001 From: Tony <3987237+random1223@users.noreply.github.com> Date: Mon, 9 Sep 2024 18:56:18 -0700 Subject: [PATCH 731/763] Update README.md Add Codety Scanner into the static analysis solution list. Here are the examples of the result: * Codety's pull request code review example: https://github.com/codetyio/codety-scanner/pull/66#issuecomment-2339438925 * Codety's GitHub code scan result example : https://github.com/codetyio/codety-scanner/runs/29907371258 Codety Scanner is open source: https://github.com/codetyio/codety-scanner --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c200a3e..9b776cf 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,7 @@ Services and platforms that have ShellCheck pre-installed and ready to use: * [Codacy](https://www.codacy.com/) * [Code Climate](https://codeclimate.com/) * [Code Factor](https://www.codefactor.io/) +* [Codety](https://www.codety.io/) via the [Codety Scanner](https://github.com/codetyio/codety-scanner) * [CircleCI](https://circleci.com) via the [ShellCheck Orb](https://circleci.com/orbs/registry/orb/circleci/shellcheck) * [Github](https://github.com/features/actions) (only Linux) * [Trunk Check](https://trunk.io/products/check) (universal linter; [allows you to explicitly version your shellcheck install](https://github.com/trunk-io/plugins/blob/bcbb361dcdbe4619af51ea7db474d7fb87540d20/.trunk/trunk.yaml#L32)) via the [shellcheck plugin](https://github.com/trunk-io/plugins/blob/main/linters/shellcheck/plugin.yaml) From 5e3e98bcb0ca594185e9e675dac929aa053dd223 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 27 Oct 2024 15:43:30 -0700 Subject: [PATCH 732/763] Use CFG to determine use-before-define for SC2218 (fixes #3070) --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 48 ++++++++++++++++++------------------- 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 43db00c..982036d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ ### Changed - SC2015 about `A && B || C` no longer triggers when B is a test command. ### Fixed +- SC2218 about function use-before-define is now more accurate. - SC2317 about unreachable commands is now less spammy for nested ones. - SC2292, optional suggestion for [[ ]], now triggers for Busybox. diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index ce11884..7f7a572 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -3765,32 +3765,32 @@ prop_checkUseBeforeDefinition1 = verifyTree checkUseBeforeDefinition "f; f() { t prop_checkUseBeforeDefinition2 = verifyNotTree checkUseBeforeDefinition "f() { true; }; f" prop_checkUseBeforeDefinition3 = verifyNotTree checkUseBeforeDefinition "if ! mycmd --version; then mycmd() { true; }; fi" prop_checkUseBeforeDefinition4 = verifyNotTree checkUseBeforeDefinition "mycmd || mycmd() { f; }" -checkUseBeforeDefinition _ t = - execWriter $ evalStateT (mapM_ examine $ revCommands) Map.empty +prop_checkUseBeforeDefinition5 = verifyTree checkUseBeforeDefinition "false || mycmd; mycmd() { f; }" +prop_checkUseBeforeDefinition6 = verifyNotTree checkUseBeforeDefinition "f() { one; }; f; f() { two; }; f" +checkUseBeforeDefinition :: Parameters -> Token -> [TokenComment] +checkUseBeforeDefinition params t = fromMaybe [] $ do + cfga <- cfgAnalysis params + let funcs = execState (doAnalysis findFunction t) Map.empty + -- Green cut: no point enumerating commands if there are no functions. + guard . not $ Map.null funcs + return $ execWriter $ doAnalysis (findInvocation cfga funcs) t where - examine t = case t of - T_Pipeline _ _ [T_Redirecting _ _ (T_Function _ _ _ name _)] -> - modify $ Map.insert name t - T_Annotation _ _ w -> examine w - T_Pipeline _ _ cmds -> do - m <- get - unless (Map.null m) $ - mapM_ (checkUsage m) $ concatMap recursiveSequences cmds - _ -> return () + findFunction t = + case t of + T_Function id _ _ name _ -> modify (Map.insertWith (++) name [id]) + _ -> return () - checkUsage map cmd = sequence_ $ do - name <- getCommandName cmd - def <- Map.lookup name map - return $ - err (getId cmd) 2218 - "This function is only defined later. Move the definition up." - - revCommands = reverse $ concat $ getCommandSequences t - recursiveSequences x = - let list = concat $ getCommandSequences x in - if null list - then [x] - else concatMap recursiveSequences list + findInvocation cfga funcs t = + case t of + T_SimpleCommand id _ (cmd:_) -> sequence_ $ do + name <- getLiteralString cmd + invocations <- Map.lookup name funcs + -- Is the function definitely being defined later? + guard $ any (\c -> CF.doesPostDominate cfga c id) invocations + -- Was one already defined, so it's actually a re-definition? + guard . not $ any (\c -> CF.doesPostDominate cfga id c) invocations + return $ err id 2218 "This function is only defined later. Move the definition up." + _ -> return () prop_checkForLoopGlobVariables1 = verify checkForLoopGlobVariables "for i in $var/*.txt; do true; done" prop_checkForLoopGlobVariables2 = verifyNot checkForLoopGlobVariables "for i in \"$var\"/*.txt; do true; done" From f2932ebcdc51527ca439737465e35b0b039a51b7 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 27 Oct 2024 16:02:56 -0700 Subject: [PATCH 733/763] Remember to add changelog to release messages (fixes #3051) --- test/check_release | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/check_release b/test/check_release index 665b265..f3ea9df 100755 --- a/test/check_release +++ b/test/check_release @@ -50,6 +50,11 @@ then fail "Expected git log message to be 'Stable version ...'" fi +if [[ $(git log -1 --pretty=%B) != *"CHANGELOG"* ]] +then + fail "Expected git log message to contain CHANGELOG" +fi + i=1 j=1 cat << EOF From 097018754b313a102834ec389079ee04673f68fa Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 27 Oct 2024 18:10:00 -0700 Subject: [PATCH 734/763] Mention that SC2002 (UUOC) is now no longer enabled by default. --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 982036d..612068d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ - SC2330: Warn about unsupported glob matches with [[ .. ]] in BusyBox. - Precompiled binaries for Linux riscv64 (linux.riscv64) ### Changed +- SC2002 about Useless Use Of Cat is now disabled by default. It can be + re-enabled with `--enable=useless-use-of-cat` or equivalent directive. - SC2015 about `A && B || C` no longer triggers when B is a test command. ### Fixed - SC2218 about function use-before-define is now more accurate. From 792466bc22ea9528313b5224621610551216e4a6 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 3 Nov 2024 13:56:51 -0800 Subject: [PATCH 735/763] Update Diff dependency (fixes #3075) --- ShellCheck.cabal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ShellCheck.cabal b/ShellCheck.cabal index fc52b12..0f604a9 100644 --- a/ShellCheck.cabal +++ b/ShellCheck.cabal @@ -53,7 +53,7 @@ library bytestring >= 0.10.6 && < 0.13, containers >= 0.5.6 && < 0.8, deepseq >= 1.4.1 && < 1.6, - Diff >= 0.4.0 && < 0.6, + Diff >= 0.4.0 && < 1.1, fgl (>= 5.7.0 && < 5.8.1.0) || (>= 5.8.1.1 && < 5.9), filepath >= 1.4.0 && < 1.5, mtl >= 2.2.2 && < 2.4, From 0ee46a0f33ebafde128e2c93dd45f2757de4d4ec Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 3 Nov 2024 14:19:08 -0800 Subject: [PATCH 736/763] Update filepath dependency --- ShellCheck.cabal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ShellCheck.cabal b/ShellCheck.cabal index 0f604a9..f31ead3 100644 --- a/ShellCheck.cabal +++ b/ShellCheck.cabal @@ -55,7 +55,7 @@ library deepseq >= 1.4.1 && < 1.6, Diff >= 0.4.0 && < 1.1, fgl (>= 5.7.0 && < 5.8.1.0) || (>= 5.8.1.1 && < 5.9), - filepath >= 1.4.0 && < 1.5, + filepath >= 1.4.0 && < 1.6, mtl >= 2.2.2 && < 2.4, parsec >= 3.1.14 && < 3.2, QuickCheck >= 2.14.2 && < 2.15, From 47bff1d5fdc478a3bfb32ffb532d33bab0e64b2c Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 3 Nov 2024 16:54:45 -0800 Subject: [PATCH 737/763] Add 24.04 to distrotest LTS --- test/distrotest | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/distrotest b/test/distrotest index ef467b8..c52a5c9 100755 --- a/test/distrotest +++ b/test/distrotest @@ -74,11 +74,12 @@ fedora:latest dnf install -y cabal-install ghc-template-haskell-devel fi archlinux:latest pacman -S -y --noconfirm cabal-install ghc-static base-devel # Ubuntu LTS +ubuntu:24.04 apt-get update && apt-get install -y cabal-install ubuntu:22.04 apt-get update && apt-get install -y cabal-install ubuntu:20.04 apt-get update && apt-get install -y cabal-install # Stack on Ubuntu LTS -ubuntu:22.04 set -e; apt-get update && apt-get install -y curl && curl -sSL https://get.haskellstack.org/ | sh -s - -f && cd /mnt && exec test/stacktest +ubuntu:24.04 set -e; apt-get update && apt-get install -y curl && curl -sSL https://get.haskellstack.org/ | sh -s - -f && cd /mnt && exec test/stacktest EOF exit "$final" From 944d87915a191af442b3895ef5af89ffb65789d8 Mon Sep 17 00:00:00 2001 From: Evan Silberman Date: Mon, 11 Nov 2024 11:24:21 -0800 Subject: [PATCH 738/763] Recognize "oksh" executable name as ksh A portable version of OpenBSD's ksh is distributed with the executable name oksh [1]. It's a descendant of pdksh and can be shellchecked as ksh. [1]: https://github.com/ibara/oksh --- src/ShellCheck/Data.hs | 1 + src/ShellCheck/Parser.hs | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Data.hs b/src/ShellCheck/Data.hs index 3000a99..a145684 100644 --- a/src/ShellCheck/Data.hs +++ b/src/ShellCheck/Data.hs @@ -167,6 +167,7 @@ shellForExecutable name = "ksh" -> return Ksh "ksh88" -> return Ksh "ksh93" -> return Ksh + "oksh" -> return Ksh _ -> Nothing flagsForRead = "sreu:n:N:i:p:a:t:" diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 66d62ff..3986dab 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -3387,7 +3387,8 @@ readScriptFile sourced = do "busybox sh", "bash", "bats", - "ksh" + "ksh", + "oksh" ] badShells = [ "awk", From 7f3f014d49d4bc6b979ed5508b1c57874e795ee8 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 28 Nov 2024 11:51:22 -0800 Subject: [PATCH 739/763] Allow latest QuickCheck --- ShellCheck.cabal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ShellCheck.cabal b/ShellCheck.cabal index f31ead3..68c32d9 100644 --- a/ShellCheck.cabal +++ b/ShellCheck.cabal @@ -58,7 +58,7 @@ library filepath >= 1.4.0 && < 1.6, mtl >= 2.2.2 && < 2.4, parsec >= 3.1.14 && < 3.2, - QuickCheck >= 2.14.2 && < 2.15, + QuickCheck >= 2.14.2 && < 2.16, regex-tdfa >= 1.2.0 && < 1.4, transformers >= 0.4.2 && < 0.7, From 3c75d82db571aa3fd337e04077daa3c01e0c878e Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Fri, 29 Nov 2024 12:58:56 -0800 Subject: [PATCH 740/763] Fix stacktest complaining about permissions on /mnt --- test/distrotest | 6 +++--- test/stacktest | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/distrotest b/test/distrotest index c52a5c9..128ee44 100755 --- a/test/distrotest +++ b/test/distrotest @@ -17,13 +17,13 @@ and is still highly experimental. Make sure you're plugged in and have screen/tmux in place, then re-run with $0 --run to continue. -Also note that dist* will be deleted. +Also note that dist*/ and .stack-work/ will be deleted. EOF exit 0 } -echo "Deleting 'dist' and 'dist-newstyle'..." -rm -rf dist dist-newstyle +echo "Deleting 'dist', 'dist-newstyle', and '.stack-work'..." +rm -rf dist dist-newstyle .stack-work execs=$(find . -name shellcheck) diff --git a/test/stacktest b/test/stacktest index 9eb8d1e..b486c31 100755 --- a/test/stacktest +++ b/test/stacktest @@ -15,7 +15,7 @@ die() { echo "$*" >&2; exit 1; } command -v stack || die "stack is missing" -stack setup || die "Failed to setup with default resolver" +stack setup --allow-different-user || die "Failed to setup with default resolver" stack build --test || die "Failed to build/test with default resolver" # Nice to haves, but not necessary From 195b70db8c697e44b65beeb83abe4c283cd4cda4 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Fri, 13 Dec 2024 23:06:49 -0500 Subject: [PATCH 741/763] Use unless instead of when and not --- src/ShellCheck/Checks/ShellSupport.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 2ec4cf7..f228832 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -330,7 +330,7 @@ checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do | t `isCommand` "echo" && argString `matches` flagRegex = if isBusyboxSh then - when (not (argString `matches` busyboxFlagRegex)) $ + unless (argString `matches` busyboxFlagRegex) $ warnMsg (getId arg) 3036 "echo flags besides -n and -e" else if isDash then From 0ecaf2b5f165497e461a47330ebbe11b05d8eb1a Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Fri, 13 Dec 2024 23:19:36 -0500 Subject: [PATCH 742/763] Use foldr instead of explicit recursion --- src/ShellCheck/Analytics.hs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 7f7a572..3a47ed9 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -5074,10 +5074,9 @@ checkExpansionWithRedirection params t = (T_Pipeline _ _ t@(_:_)) -> checkCmd id (last t) _ -> return () - checkCmd captureId (T_Redirecting _ redirs _) = walk captureId redirs + checkCmd captureId (T_Redirecting _ redirs _) = foldr (walk captureId) (return ()) redirs - walk captureId [] = return () - walk captureId (t:rest) = + walk captureId t acc = case t of T_FdRedirect _ _ (T_IoDuplicate _ _ "1") -> return () T_FdRedirect id "1" (T_IoDuplicate _ _ _) -> return () @@ -5086,7 +5085,7 @@ checkExpansionWithRedirection params t = if getLiteralString file == Just "/dev/null" then emit id captureId False else emit id captureId True - _ -> walk captureId rest + _ -> acc emit redirectId captureId suggestTee = do warn captureId 2327 "This command substitution will be empty because the command's output gets redirected away." From 5adfea21eec84a6bc8bbcaf4e22c02461dd850d9 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Fri, 13 Dec 2024 23:20:48 -0500 Subject: [PATCH 743/763] Use the result of the comparison directly instead of an if/else --- src/ShellCheck/Analytics.hs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 3a47ed9..4757e57 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -5082,9 +5082,7 @@ checkExpansionWithRedirection params t = T_FdRedirect id "1" (T_IoDuplicate _ _ _) -> return () T_FdRedirect id "" (T_IoDuplicate _ op _) | op `elem` [T_GREATAND (Id 0), T_Greater (Id 0)] -> emit id captureId True T_FdRedirect id str (T_IoFile _ op file) | str `elem` ["", "1"] && op `elem` [ T_DGREAT (Id 0), T_Greater (Id 0) ] -> - if getLiteralString file == Just "/dev/null" - then emit id captureId False - else emit id captureId True + emit id captureId $ getLiteralString file /= Just "/dev/null" _ -> acc emit redirectId captureId suggestTee = do From 26b949b9b0b1552639fdf24411096cf00749be18 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Fri, 13 Dec 2024 23:45:32 -0500 Subject: [PATCH 744/763] Use mapM_ instead of isJust and fromJust --- src/ShellCheck/Checks/Commands.hs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index c10016e..c37a67d 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -1431,9 +1431,8 @@ prop_checkBackreferencingDeclaration7 = verify (checkBackreferencingDeclaration checkBackreferencingDeclaration cmd = CommandCheck (Exactly cmd) check where check t = do - cfga <- asks cfgAnalysis - when (isJust cfga) $ - foldM_ (perArg $ fromJust cfga) M.empty $ arguments t + maybeCfga <- asks cfgAnalysis + mapM_ (\cfga -> foldM_ (perArg cfga) M.empty $ arguments t) maybeCfga perArg cfga leftArgs t = case t of From 7deb7e853b4bc5d2ad90f021de180055e87611b5 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Fri, 13 Dec 2024 23:47:55 -0500 Subject: [PATCH 745/763] Use mapM_ instead of sequence_ and <$> --- src/ShellCheck/Checks/ControlFlow.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Checks/ControlFlow.hs b/src/ShellCheck/Checks/ControlFlow.hs index d23fa15..9f63141 100644 --- a/src/ShellCheck/Checks/ControlFlow.hs +++ b/src/ShellCheck/Checks/ControlFlow.hs @@ -78,7 +78,7 @@ controlFlowEffectChecks = [ runNodeChecks :: [ControlFlowNodeCheck] -> ControlFlowCheck runNodeChecks perNode = do cfg <- asks cfgAnalysis - sequence_ $ runOnAll <$> cfg + mapM_ runOnAll cfg where getData datas n@(node, label) = do (pre, post) <- M.lookup node datas From d3001f337aa3f7653a621b302261f4eac01890d0 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Fri, 13 Dec 2024 23:57:50 -0500 Subject: [PATCH 746/763] Simplify getParseOutput --- src/ShellCheck/Parser.hs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 66d62ff..9628b2e 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -3451,12 +3451,12 @@ isNotOk p s = parsesCleanly p s == Nothing -- The string does not parse -- If the parser matches the string, return Right [ParseNotes+ParseProblems] -- If it does not match the string, return Left [ParseProblems] getParseOutput parser string = runIdentity $ do - (res, sys) <- runParser testEnvironment - (parser >> eof >> getState) "-" string - case (res, sys) of - (Right userState, systemState) -> - return $ Right $ parseNotes userState ++ parseProblems systemState - (Left _, systemState) -> return $ Left $ parseProblems systemState + (res, systemState) <- runParser testEnvironment + (parser >> eof >> getState) "-" string + return $ case res of + Right userState -> + Right $ parseNotes userState ++ parseProblems systemState + Left _ -> Left $ parseProblems systemState -- If the parser matches the string, return Just whether it was clean (without emitting suggestions) -- Otherwise, Nothing From fe315a25c42219b8a7b0b16ffa69351421e2e997 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lawrence=20Vel=C3=A1zquez?= Date: Sat, 28 Dec 2024 03:05:07 -0500 Subject: [PATCH 747/763] Recognize internal variables new in bash 5.3 From the bug-bash@gnu.org announcement "Bash-5.3-beta available": q. GLOBSORT: new variable to specify how to sort the results of pathname expansion (name, size, blocks, mtime, atime, ctime, none) in ascending or descending order. w. BASH_MONOSECONDS: new dynamic variable that returns the value of the system's monotonic clock, if one is available. x. BASH_TRAPSIG: new variable, set to the numeric signal number of the trap being executed while it's running. https://lists.gnu.org/archive/html/bug-bash/2024-12/msg00120.html --- src/ShellCheck/Data.hs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Data.hs b/src/ShellCheck/Data.hs index 3000a99..3876507 100644 --- a/src/ShellCheck/Data.hs +++ b/src/ShellCheck/Data.hs @@ -49,6 +49,7 @@ internalVariables = [ "LINES", "MAIL", "MAILCHECK", "MAILPATH", "OPTERR", "PATH", "POSIXLY_CORRECT", "PROMPT_COMMAND", "PROMPT_DIRTRIM", "PS0", "PS1", "PS2", "PS3", "PS4", "SHELL", "TIMEFORMAT", "TMOUT", "TMPDIR", + "BASH_MONOSECONDS", "BASH_TRAPSIG", "GLOBSORT", "auto_resume", "histchars", -- Other @@ -78,7 +79,7 @@ variablesWithoutSpaces = specialVariablesWithoutSpaces ++ [ "EPOCHREALTIME", "EPOCHSECONDS", "LINENO", "OPTIND", "PPID", "RANDOM", "READLINE_ARGUMENT", "READLINE_MARK", "READLINE_POINT", "SECONDS", "SHELLOPTS", "SHLVL", "SRANDOM", "UID", "COLUMNS", "HISTFILESIZE", - "HISTSIZE", "LINES" + "HISTSIZE", "LINES", "BASH_MONOSECONDS", "BASH_TRAPSIG" -- shflags , "FLAGS_ERROR", "FLAGS_FALSE", "FLAGS_TRUE" From cbf0b33463601fbd9827db31e08db424e3381074 Mon Sep 17 00:00:00 2001 From: Adrian Fluturel Date: Tue, 7 Jan 2025 03:24:29 +0100 Subject: [PATCH 748/763] Skip SC2015 when the last command is true --- src/ShellCheck/Analytics.hs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 4757e57..1329c86 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -881,13 +881,15 @@ prop_checkShorthandIf6 = verifyNot checkShorthandIf "if foo && bar || baz; then prop_checkShorthandIf7 = verifyNot checkShorthandIf "while foo && bar || baz; do true; done" prop_checkShorthandIf8 = verify checkShorthandIf "if true; then foo && bar || baz; fi" prop_checkShorthandIf9 = verifyNot checkShorthandIf "foo && [ -x /file ] || bar" +prop_checkShorthandIf10 = verifyNot checkShorthandIf "foo && bar || true" +prop_checkShorthandIf11 = verify checkShorthandIf "foo && bar || false" checkShorthandIf params x@(T_OrIf _ (T_AndIf id _ b) (T_Pipeline _ _ t)) | not (isOk t || inCondition) && not (isTestCommand b) = info id 2015 "Note that A && B || C is not if-then-else. C may run when A is true." where isOk [t] = isAssignment t || fromMaybe False (do name <- getCommandBasename t - return $ name `elem` ["echo", "exit", "return", "printf"]) + return $ name `elem` ["echo", "exit", "return", "printf", "true"]) isOk _ = False inCondition = isCondition $ getPath (parentMap params) x checkShorthandIf _ _ = return () From 3a9ddae06b7e2293ebf61eaf8c71b01bbb769614 Mon Sep 17 00:00:00 2001 From: Eisuke Kawashima Date: Mon, 24 Mar 2025 05:49:06 +0900 Subject: [PATCH 749/763] fix(SC3013)!: remove SC3013 since the operators are specified by POSIX.1-2024 https://pubs.opengroup.org/onlinepubs/9799919799/utilities/test.html fix #3167 --- CHANGELOG.md | 2 ++ src/ShellCheck/Checks/ShellSupport.hs | 8 +------- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 612068d..1a24b53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ - SC2317 about unreachable commands is now less spammy for nested ones. - SC2292, optional suggestion for [[ ]], now triggers for Busybox. +### Removed +- SC3013: removed since the operators `-op/-nt/-ef` are specified in POSIX.1-2024 ## v0.10.0 - 2024-03-07 ### Added diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index f228832..1789b6f 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -86,7 +86,7 @@ checkForDecimals = ForShell [Sh, Dash, BusyboxSh, Bash] f prop_checkBashisms = verify checkBashisms "while read a; do :; done < <(a)" -prop_checkBashisms2 = verify checkBashisms "[ foo -nt bar ]" +prop_checkBashisms2 = verifyNot checkBashisms "[ foo -nt bar ]" prop_checkBashisms3 = verify checkBashisms "echo $((i++))" prop_checkBashisms4 = verify checkBashisms "rm !(*.hs)" prop_checkBashisms5 = verify checkBashisms "source file" @@ -252,12 +252,6 @@ checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do bashism (T_SimpleCommand id _ [asStr -> Just "test", lhs, asStr -> Just op, rhs]) | op `elem` [ "<", ">", "\\<", "\\>", "<=", ">=", "\\<=", "\\>="] = unless isDash $ warnMsg id 3012 $ "lexicographical " ++ op ++ " is" - bashism (TC_Binary id SingleBracket op _ _) - | op `elem` [ "-ot", "-nt", "-ef" ] = - unless isDash $ warnMsg id 3013 $ op ++ " is" - bashism (T_SimpleCommand id _ [asStr -> Just "test", lhs, asStr -> Just op, rhs]) - | op `elem` [ "-ot", "-nt", "-ef" ] = - unless isDash $ warnMsg id 3013 $ op ++ " is" bashism (TC_Binary id SingleBracket "==" _ _) = unless isBusyboxSh $ warnMsg id 3014 "== in place of = is" bashism (T_SimpleCommand id _ [asStr -> Just "test", lhs, asStr -> Just "==", rhs]) = From bc60607f9e5be0ea7528589410845aef2d6f10a3 Mon Sep 17 00:00:00 2001 From: Eisuke Kawashima Date: Mon, 24 Mar 2025 06:04:19 +0900 Subject: [PATCH 750/763] fix(SC3012)!: do not warn about `\<` and `\>` in test/[] as specified in POSIX.1-2024 https://pubs.opengroup.org/onlinepubs/9799919799/utilities/test.html fix #3168 --- CHANGELOG.md | 1 + src/ShellCheck/Checks/ShellSupport.hs | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 612068d..82630bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - SC2002 about Useless Use Of Cat is now disabled by default. It can be re-enabled with `--enable=useless-use-of-cat` or equivalent directive. - SC2015 about `A && B || C` no longer triggers when B is a test command. +- SC3012: Do not warn about `\<` and `\>` in test/[] as specified in POSIX.1-2024 ### Fixed - SC2218 about function use-before-define is now more accurate. - SC2317 about unreachable commands is now less spammy for nested ones. diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index f228832..2a34931 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -247,10 +247,10 @@ checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do unless isBusyboxSh $ warnMsg id 3010 "[[ ]] is" bashism (T_HereString id _) = warnMsg id 3011 "here-strings are" bashism (TC_Binary id SingleBracket op _ _) - | op `elem` [ "<", ">", "\\<", "\\>", "<=", ">=", "\\<=", "\\>="] = + | op `elem` [ "<", ">", "<=", ">=", "\\<=", "\\>="] = unless isDash $ warnMsg id 3012 $ "lexicographical " ++ op ++ " is" bashism (T_SimpleCommand id _ [asStr -> Just "test", lhs, asStr -> Just op, rhs]) - | op `elem` [ "<", ">", "\\<", "\\>", "<=", ">=", "\\<=", "\\>="] = + | op `elem` [ "<", ">", "<=", ">=", "\\<=", "\\>="] = unless isDash $ warnMsg id 3012 $ "lexicographical " ++ op ++ " is" bashism (TC_Binary id SingleBracket op _ _) | op `elem` [ "-ot", "-nt", "-ef" ] = From 4f628cbe2a617e76fe3686a79c3d1eaf443acc08 Mon Sep 17 00:00:00 2001 From: Eisuke Kawashima Date: Fri, 4 Apr 2025 17:31:07 +0900 Subject: [PATCH 751/763] feat: check tautologically-false conditionals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - fix #3179 — negation of SC2055, `[ x = y -a x = z]` - fix #3181 — negation of SC2056, `(( x == y && x == z ))` - fix #3180 — negation of SC2252, `[ x = y ] && [ x = z ]` --- src/ShellCheck/Analytics.hs | 50 +++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 4757e57..431e56b 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -123,6 +123,7 @@ nodeChecks = [ ,checkCaseAgainstGlob ,checkCommarrays ,checkOrNeq + ,checkAndEq ,checkEchoWc ,checkConstantIfs ,checkPipedAssignment @@ -1631,6 +1632,55 @@ checkOrNeq _ (T_OrIf id lhs rhs) = sequence_ $ do checkOrNeq _ _ = return () +prop_checkAndEq1 = verify checkAndEq "if [[ $lol -eq cow && $lol -eq foo ]]; then echo foo; fi" +prop_checkAndEq2 = verify checkAndEq "(( a==lol && a==foo ))" +prop_checkAndEq3 = verify checkAndEq "[ \"$a\" = lol && \"$a\" = foo ]" +prop_checkAndEq4 = verifyNot checkAndEq "[ a = $cow && b = $foo ]" +prop_checkAndEq5 = verifyNot checkAndEq "[[ $a = /home && $a = */public_html/* ]]" +prop_checkAndEq6 = verify checkAndEq "[ $a = a ] && [ $a = b ]" +prop_checkAndEq7 = verify checkAndEq "[ $a = a ] && [ $a = b ] || true" +prop_checkAndEq8 = verifyNot checkAndEq "[[ $a == x && $a == x ]]" +prop_checkAndEq9 = verifyNot checkAndEq "[ 0 -eq $FOO ] && [ 0 -eq $BAR ]" + +-- For test-level "and": [ x = y -a x = z ] +checkAndEq _ (TC_And id typ op (TC_Binary _ _ op1 lhs1 rhs1 ) (TC_Binary _ _ op2 lhs2 rhs2)) + | (op1 == op2 && (op1 == "-eq" || op1 == "=" || op1 == "==")) && lhs1 == lhs2 && rhs1 /= rhs2 && not (any isGlob [rhs1,rhs2]) = + warn id 2055 $ "You probably wanted " ++ (if typ == SingleBracket then "-o" else "||") ++ " here, otherwise it's always false." + +-- For arithmetic context "and" +checkAndEq _ (TA_Binary id "&&" (TA_Binary _ "==" word1 _) (TA_Binary _ "==" word2 _)) + | word1 == word2 = + warn id 2056 "You probably wanted || here, otherwise it's always false." + +-- For command level "and": [ x = y ] && [ x = z ] +checkAndEq _ (T_AndIf id lhs rhs) = sequence_ $ do + (lhs1, op1, rhs1) <- getExpr lhs + (lhs2, op2, rhs2) <- getExpr rhs + guard $ op1 == op2 && op1 `elem` ["-eq", "=", "=="] + guard $ lhs1 == lhs2 && rhs1 /= rhs2 + guard . not $ any isGlob [rhs1, rhs2] + return $ warn id 2252 "You probably wanted || here, otherwise it's always false." + where + getExpr x = + case x of + T_AndIf _ lhs _ -> getExpr lhs -- Fetches x and y in `T_AndIf x (T_AndIf y z)` + T_Pipeline _ _ [x] -> getExpr x + T_Redirecting _ _ c -> getExpr c + T_Condition _ _ c -> getExpr c + TC_Binary _ _ op lhs rhs -> orient (lhs, op, rhs) + _ -> Nothing + + -- Swap items so that the constant side is rhs (or Nothing if both/neither is constant) + orient (lhs, op, rhs) = + case (isConstant lhs, isConstant rhs) of + (True, False) -> return (rhs, op, lhs) + (False, True) -> return (lhs, op, rhs) + _ -> Nothing + + +checkAndEq _ _ = return () + + prop_checkValidCondOps1 = verify checkValidCondOps "[[ a -xz b ]]" prop_checkValidCondOps2 = verify checkValidCondOps "[ -M a ]" prop_checkValidCondOps2a = verifyNot checkValidCondOps "[ 3 \\> 2 ]" From 8ff0c5be7a85561d23aea762f0495c144896aee3 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 6 Apr 2025 19:26:54 -0700 Subject: [PATCH 752/763] Suppress SC2216 when piping to cp/mv/rm -i (fixes #3141). --- src/ShellCheck/Analytics.hs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 4757e57..86a3292 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -3596,6 +3596,8 @@ prop_checkPipeToNowhere17 = verify checkPipeToNowhere "echo World | cat << 'EOF' prop_checkPipeToNowhere18 = verifyNot checkPipeToNowhere "ls 1>&3 3>&1 3>&- | wc -l" prop_checkPipeToNowhere19 = verifyNot checkPipeToNowhere "find . -print0 | du --files0-from=/dev/stdin" prop_checkPipeToNowhere20 = verifyNot checkPipeToNowhere "find . | du --exclude-from=/dev/fd/0" +prop_checkPipeToNowhere21 = verifyNot checkPipeToNowhere "yes | cp -ri foo/* bar" +prop_checkPipeToNowhere22 = verifyNot checkPipeToNowhere "yes | rm --interactive *" data PipeType = StdoutPipe | StdoutStderrPipe | NoPipe deriving (Eq) checkPipeToNowhere :: Parameters -> Token -> WriterT [TokenComment] Identity () @@ -3661,6 +3663,7 @@ checkPipeToNowhere params t = commandSpecificException name cmd = case name of "du" -> any ((`elem` ["exclude-from", "files0-from"]) . snd) $ getAllFlags cmd + _ | name `elem` interactiveFlagCmds -> hasInteractiveFlag cmd _ -> False warnAboutDupes (n, list@(_:_:_)) = @@ -3684,7 +3687,7 @@ checkPipeToNowhere params t = name <- getCommandBasename cmd guard $ name `elem` nonReadingCommands guard . not $ hasAdditionalConsumers cmd - guard . not $ name `elem` ["cp", "mv", "rm"] && cmd `hasFlag` "i" + guard . not $ name `elem` interactiveFlagCmds && hasInteractiveFlag cmd let suggestion = if name == "echo" then "Did you want 'cat' instead?" @@ -3699,6 +3702,9 @@ checkPipeToNowhere params t = treeContains pred t = isNothing $ doAnalysis (guard . not . pred) t + interactiveFlagCmds = [ "cp", "mv", "rm" ] + hasInteractiveFlag cmd = cmd `hasFlag` "i" || cmd `hasFlag` "interactive" + mayConsume t = case t of T_ProcSub _ "<" _ -> True From 72af76f443b81bbcd10889df8caf19be1671f7a2 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 6 Apr 2025 19:58:13 -0700 Subject: [PATCH 753/763] Supress SC2093 when execfail is set (fixes #3178) --- src/ShellCheck/Analytics.hs | 4 +++- src/ShellCheck/AnalyzerLib.hs | 6 ++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 86a3292..e6a1fd6 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1896,7 +1896,9 @@ prop_checkSpuriousExec8 = verifyNot checkSpuriousExec "exec {origout}>&1- >tmp.l prop_checkSpuriousExec9 = verify checkSpuriousExec "for file in rc.d/*; do exec \"$file\"; done" prop_checkSpuriousExec10 = verifyNot checkSpuriousExec "exec file; r=$?; printf >&2 'failed\n'; return $r" prop_checkSpuriousExec11 = verifyNot checkSpuriousExec "exec file; :" -checkSpuriousExec _ = doLists +prop_checkSpuriousExec12 = verifyNot checkSpuriousExec "#!/bin/bash\nshopt -s execfail; exec foo; exec bar; echo 'Error'; exit 1;" +prop_checkSpuriousExec13 = verify checkSpuriousExec "#!/bin/dash\nshopt -s execfail; exec foo; exec bar; echo 'Error'; exit 1;" +checkSpuriousExec params t = when (not $ hasExecfail params) $ doLists t where doLists (T_Script _ _ cmds) = doList cmds False doLists (T_BraceGroup _ cmds) = doList cmds False diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 531ce8b..da528a4 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -89,6 +89,8 @@ data Parameters = Parameters { hasSetE :: Bool, -- Whether this script has 'set -o pipefail' anywhere. hasPipefail :: Bool, + -- Whether this script has 'shopt -s execfail' anywhere. + hasExecfail :: Bool, -- A linear (bad) analysis of data flow variableFlow :: [StackData], -- A map from Id to Token @@ -226,6 +228,10 @@ makeParameters spec = params BusyboxSh -> isOptionSet "pipefail" root Sh -> True Ksh -> isOptionSet "pipefail" root, + hasExecfail = + case shellType params of + Bash -> isOptionSet "execfail" root + _ -> False, shellTypeSpecified = isJust (asShellType spec) || isJust (asFallbackShell spec), idMap = getTokenMap root, parentMap = getParentTree root, From e4853af5b0f541d8070d9c76adb59ccd9b1b44f0 Mon Sep 17 00:00:00 2001 From: Eisuke Kawashima Date: Fri, 21 Mar 2025 18:49:06 +0900 Subject: [PATCH 754/763] doc: update man --- shellcheck.1.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/shellcheck.1.md b/shellcheck.1.md index b873e45..c768bfe 100644 --- a/shellcheck.1.md +++ b/shellcheck.1.md @@ -78,7 +78,7 @@ not warn at all, as `ksh` supports decimals in arithmetic contexts. : Don't try to look for .shellcheckrc configuration files. ---rcfile\ RCFILE +**--rcfile** *RCFILE* : Prefer the specified configuration file over searching for one in the default locations. @@ -317,7 +317,7 @@ Here is an example `.shellcheckrc`: disable=SC2236 If no `.shellcheckrc` is found in any of the parent directories, ShellCheck -will look in `~/.shellcheckrc` followed by the XDG config directory +will look in `~/.shellcheckrc` followed by the `$XDG_CONFIG_HOME` (usually `~/.config/shellcheckrc`) on Unix, or `%APPDATA%/shellcheckrc` on Windows. Only the first file found will be used. @@ -403,4 +403,4 @@ see https://gnu.org/licenses/gpl.html # SEE ALSO -sh(1) bash(1) +sh(1) bash(1) dash(1) ksh(1) From 574c6d18fbbae18b65b244ce2b37c3dced452a5a Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 8 Apr 2025 10:23:10 -0700 Subject: [PATCH 755/763] Suggest using test -e instead of -a (fixes #3174). --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 612068d..6309192 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ - SC2327/SC2328: Warn about capturing the output of redirected commands. - SC2329: Warn when (non-escaping) functions are never invoked. - SC2330: Warn about unsupported glob matches with [[ .. ]] in BusyBox. +- SC2331: Suggest using standard -e instead of unary -a in tests. - Precompiled binaries for Linux riscv64 (linux.riscv64) ### Changed - SC2002 about Useless Use Of Cat is now disabled by default. It can be diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index e6a1fd6..4984d44 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -204,6 +204,7 @@ nodeChecks = [ ,checkUnnecessaryParens ,checkPlusEqualsNumber ,checkExpansionWithRedirection + ,checkUnaryTestA ] optionalChecks = map fst optionalTreeChecks @@ -5098,6 +5099,15 @@ checkExpansionWithRedirection params t = err redirectId 2328 $ "This redirection takes output away from the command substitution" ++ if suggestTee then " (use tee to duplicate)." else "." +prop_checkUnaryTestA1 = verify checkUnaryTestA "[ -a foo ]" +prop_checkUnaryTestA2 = verify checkUnaryTestA "[ ! -a foo ]" +prop_checkUnaryTestA3 = verifyNot checkUnaryTestA "[ foo -a bar ]" +checkUnaryTestA params t = + case t of + TC_Unary id _ "-a" _ -> + styleWithFix id 2331 "For file existence, prefer standard -e over legacy -a." $ + fixWith [replaceStart id params 2 "-e"] + _ -> return () return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) From c41f3a4b8ac4eb7bcff230928a47f8f92f15f49d Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 8 Apr 2025 10:53:52 -0700 Subject: [PATCH 756/763] Warn about [ ! -o opt ] (and -a) being unconditionally true (fixes #3174) --- CHANGELOG.md | 2 ++ src/ShellCheck/Checks/ShellSupport.hs | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6309192..bc646d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ - SC2329: Warn when (non-escaping) functions are never invoked. - SC2330: Warn about unsupported glob matches with [[ .. ]] in BusyBox. - SC2331: Suggest using standard -e instead of unary -a in tests. +- SC2332: Warn about `[ ! -o opt ]` being unconditionally true in Bash. +- SC3062: Warn about bashism `[ -o opt ]`. - Precompiled binaries for Linux riscv64 (linux.riscv64) ### Changed - SC2002 about Useless Use Of Cat is now disabled by default. It can be diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index f228832..624d474 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -63,6 +63,7 @@ checks = [ ,checkPS1Assignments ,checkMultipleBangs ,checkBangAfterPipe + ,checkNegatedUnaryOps ] testChecker (ForShell _ t) = @@ -218,6 +219,7 @@ prop_checkBashisms124 = verify checkBashisms "#!/bin/dash\ntype -p test" prop_checkBashisms125 = verifyNot checkBashisms "#!/bin/busybox sh\ntype -p test" prop_checkBashisms126 = verifyNot checkBashisms "#!/bin/busybox sh\nread -p foo -r bar" prop_checkBashisms127 = verifyNot checkBashisms "#!/bin/busybox sh\necho -ne foo" +prop_checkBashisms128 = verify checkBashisms "#!/bin/dash\ntype -p test" checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do params <- ask kludge params t @@ -272,6 +274,8 @@ checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do warnMsg id 3016 "unary -v (in place of [ -n \"${var+x}\" ]) is" bashism (TC_Unary id _ "-a" _) = warnMsg id 3017 "unary -a in place of -e is" + bashism (TC_Unary id _ "-o" _) = + warnMsg id 3062 "unary -o to check options is" bashism (T_SimpleCommand id _ [asStr -> Just "test", asStr -> Just "-a", _]) = warnMsg id 3017 "unary -a in place of -e is" bashism (TA_Unary id op _) @@ -649,5 +653,22 @@ checkBangAfterPipe = ForShell [Dash, BusyboxSh, Sh, Bash] f err id 2326 "! is not allowed in the middle of pipelines. Use command group as in cmd | { ! cmd; } if necessary." _ -> return () + +prop_checkNegatedUnaryOps1 = verify checkNegatedUnaryOps "[ ! -o braceexpand ]" +prop_checkNegatedUnaryOps2 = verifyNot checkNegatedUnaryOps "[ -o braceexpand ]" +prop_checkNegatedUnaryOps3 = verifyNot checkNegatedUnaryOps "[[ ! -o braceexpand ]]" +prop_checkNegatedUnaryOps4 = verifyNot checkNegatedUnaryOps "! [ -o braceexpand ]" +prop_checkNegatedUnaryOps5 = verify checkNegatedUnaryOps "[ ! -a file ]" +checkNegatedUnaryOps = ForShell [Bash] f + where + f token = case token of + TC_Unary id SingleBracket "!" (TC_Unary _ _ op _) | op `elem` ["-a", "-o"] -> + err id 2332 $ msg op + _ -> return () + + msg "-o" = "[ ! -o opt ] is always true because -o becomes logical OR. Use [[ ]] or ! [ -o opt ]." + msg "-a" = "[ ! -a file ] is always true because -a becomes logical AND. Use -e instead." + msg _ = pleaseReport "unhandled negated unary message" + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) From 7fc992d0dc590f32bd7265e719757102f5ad3b76 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 8 Apr 2025 20:52:52 -0700 Subject: [PATCH 757/763] Suppress SC2119/SC2120 for ${1:-default} (fixes #2023) --- src/ShellCheck/Analytics.hs | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 4984d44..85104d8 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2825,6 +2825,8 @@ prop_checkUnpassedInFunctions11 = verifyNotTree checkUnpassedInFunctions "foo() prop_checkUnpassedInFunctions12 = verifyNotTree checkUnpassedInFunctions "foo() { echo ${!var*}; }; foo;" prop_checkUnpassedInFunctions13 = verifyNotTree checkUnpassedInFunctions "# shellcheck disable=SC2120\nfoo() { echo $1; }\nfoo\n" prop_checkUnpassedInFunctions14 = verifyTree checkUnpassedInFunctions "foo() { echo $#; }; foo" +prop_checkUnpassedInFunctions15 = verifyNotTree checkUnpassedInFunctions "foo() { echo ${1-x}; }; foo" +prop_checkUnpassedInFunctions16 = verifyNotTree checkUnpassedInFunctions "foo() { echo ${1:-x}; }; foo" checkUnpassedInFunctions params root = execWriter $ mapM_ warnForGroup referenceGroups where @@ -2841,9 +2843,10 @@ checkUnpassedInFunctions params root = case x of Assignment (_, _, str, _) -> isPositional str _ -> False + isPositionalReference function x = case x of - Reference (_, t, str) -> isPositional str && t `isDirectChildOf` function + Reference (_, t, str) -> isPositional str && t `isDirectChildOf` function && not (hasDefaultValue t) _ -> False isDirectChildOf child parent = fromMaybe False $ do @@ -2857,6 +2860,7 @@ checkUnpassedInFunctions params root = referenceList :: [(String, Bool, Token)] referenceList = execWriter $ doAnalysis (sequence_ . checkCommand) root + checkCommand :: Token -> Maybe (Writer [(String, Bool, Token)] ()) checkCommand t@(T_SimpleCommand _ _ (cmd:args)) = do str <- getLiteralString cmd @@ -2867,6 +2871,21 @@ checkUnpassedInFunctions params root = isPositional str = str == "*" || str == "@" || str == "#" || (all isDigit str && str /= "0" && str /= "") + -- True if t is a variable that specifies a default value, + -- such as ${1-x} or ${1:-x}. + hasDefaultValue t = + case t of + T_DollarBraced _ True l -> + let str = concat $ oversimplify l + in isDefaultValueModifier $ getBracedModifier str + _ -> False + + isDefaultValueModifier str = + case str of + '-':_ -> True + ':':'-':_ -> True + _ -> False + isArgumentless (_, b, _) = b referenceGroups = Map.elems $ foldr updateWith Map.empty referenceList updateWith x@(name, _, _) = Map.insertWith (++) name [x] From 553a80f77ad6426107fe5fefcdc950e64c420b1d Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 8 Apr 2025 21:21:50 -0700 Subject: [PATCH 758/763] Also ignore SC2119 for :? and :+. --- src/ShellCheck/Analytics.hs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 85104d8..9eec8ed 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2827,6 +2827,8 @@ prop_checkUnpassedInFunctions13 = verifyNotTree checkUnpassedInFunctions "# shel prop_checkUnpassedInFunctions14 = verifyTree checkUnpassedInFunctions "foo() { echo $#; }; foo" prop_checkUnpassedInFunctions15 = verifyNotTree checkUnpassedInFunctions "foo() { echo ${1-x}; }; foo" prop_checkUnpassedInFunctions16 = verifyNotTree checkUnpassedInFunctions "foo() { echo ${1:-x}; }; foo" +prop_checkUnpassedInFunctions17 = verifyNotTree checkUnpassedInFunctions "foo() { mycommand ${1+--verbose}; }; foo" +prop_checkUnpassedInFunctions18 = verifyNotTree checkUnpassedInFunctions "foo() { if mycheck; then foo ${1?Missing}; fi; }; foo" checkUnpassedInFunctions params root = execWriter $ mapM_ warnForGroup referenceGroups where @@ -2882,9 +2884,10 @@ checkUnpassedInFunctions params root = isDefaultValueModifier str = case str of - '-':_ -> True - ':':'-':_ -> True + ':':c:_ -> c `elem` handlesDefault + c:_ -> c `elem` handlesDefault _ -> False + where handlesDefault = "-+?" isArgumentless (_, b, _) = b referenceGroups = Map.elems $ foldr updateWith Map.empty referenceList From efb5a5a2741e690d43d5539809326d5e249fd9f2 Mon Sep 17 00:00:00 2001 From: Eisuke Kawashima Date: Tue, 25 Mar 2025 01:31:06 +0900 Subject: [PATCH 759/763] fix(SC3013): check POSIX-compliant unary operators for test and [ fix #2125 --- src/ShellCheck/Checks/ShellSupport.hs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 624d474..2039483 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -220,6 +220,9 @@ prop_checkBashisms125 = verifyNot checkBashisms "#!/bin/busybox sh\ntype -p test prop_checkBashisms126 = verifyNot checkBashisms "#!/bin/busybox sh\nread -p foo -r bar" prop_checkBashisms127 = verifyNot checkBashisms "#!/bin/busybox sh\necho -ne foo" prop_checkBashisms128 = verify checkBashisms "#!/bin/dash\ntype -p test" +prop_checkBashisms129 = verify checkBashisms "#!/bin/sh\n[ -k /tmp ]" +prop_checkBashisms130 = verifyNot checkBashisms "#!/bin/dash\ntest -k /tmp" +prop_checkBashisms131 = verify checkBashisms "#!/bin/sh\n[ -o errexit ]" checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do params <- ask kludge params t @@ -254,6 +257,18 @@ checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do bashism (T_SimpleCommand id _ [asStr -> Just "test", lhs, asStr -> Just op, rhs]) | op `elem` [ "<", ">", "\\<", "\\>", "<=", ">=", "\\<=", "\\>="] = unless isDash $ warnMsg id 3012 $ "lexicographical " ++ op ++ " is" + bashism (TC_Unary id _ op _) + | op `elem` [ "-k", "-G", "-O" ] = + unless isDash $ warnMsg id 3013 $ op ++ " is" + bashism (T_SimpleCommand id _ [asStr -> Just "test", asStr -> Just op, _]) + | op `elem` [ "-k", "-G", "-O" ] = + unless isDash $ warnMsg id 3013 $ op ++ " is" + bashism (TC_Unary id _ op _) + | op `elem` [ "-N", "-o", "-R" ] = + warnMsg id 3013 $ op ++ " is" + bashism (T_SimpleCommand id _ [asStr -> Just "test", asStr -> Just op, _]) + | op `elem` [ "-N", "-o", "-R" ] = + warnMsg id 3013 $ op ++ " is" bashism (TC_Binary id SingleBracket op _ _) | op `elem` [ "-ot", "-nt", "-ef" ] = unless isDash $ warnMsg id 3013 $ op ++ " is" From dc41f0cc5bdcf1c814b184892b20ee0d2822e95a Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Fri, 11 Apr 2025 14:14:09 -0700 Subject: [PATCH 760/763] Refactor checks for POSIX test flags --- src/ShellCheck/Checks/ShellSupport.hs | 98 +++++++++++++++------------ 1 file changed, 56 insertions(+), 42 deletions(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 2039483..c828555 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -251,48 +251,16 @@ checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do bashism (T_Condition id DoubleBracket _) = unless isBusyboxSh $ warnMsg id 3010 "[[ ]] is" bashism (T_HereString id _) = warnMsg id 3011 "here-strings are" - bashism (TC_Binary id SingleBracket op _ _) - | op `elem` [ "<", ">", "\\<", "\\>", "<=", ">=", "\\<=", "\\>="] = - unless isDash $ warnMsg id 3012 $ "lexicographical " ++ op ++ " is" - bashism (T_SimpleCommand id _ [asStr -> Just "test", lhs, asStr -> Just op, rhs]) - | op `elem` [ "<", ">", "\\<", "\\>", "<=", ">=", "\\<=", "\\>="] = - unless isDash $ warnMsg id 3012 $ "lexicographical " ++ op ++ " is" - bashism (TC_Unary id _ op _) - | op `elem` [ "-k", "-G", "-O" ] = - unless isDash $ warnMsg id 3013 $ op ++ " is" - bashism (T_SimpleCommand id _ [asStr -> Just "test", asStr -> Just op, _]) - | op `elem` [ "-k", "-G", "-O" ] = - unless isDash $ warnMsg id 3013 $ op ++ " is" - bashism (TC_Unary id _ op _) - | op `elem` [ "-N", "-o", "-R" ] = - warnMsg id 3013 $ op ++ " is" - bashism (T_SimpleCommand id _ [asStr -> Just "test", asStr -> Just op, _]) - | op `elem` [ "-N", "-o", "-R" ] = - warnMsg id 3013 $ op ++ " is" - bashism (TC_Binary id SingleBracket op _ _) - | op `elem` [ "-ot", "-nt", "-ef" ] = - unless isDash $ warnMsg id 3013 $ op ++ " is" - bashism (T_SimpleCommand id _ [asStr -> Just "test", lhs, asStr -> Just op, rhs]) - | op `elem` [ "-ot", "-nt", "-ef" ] = - unless isDash $ warnMsg id 3013 $ op ++ " is" - bashism (TC_Binary id SingleBracket "==" _ _) = - unless isBusyboxSh $ warnMsg id 3014 "== in place of = is" - bashism (T_SimpleCommand id _ [asStr -> Just "test", lhs, asStr -> Just "==", rhs]) = - unless isBusyboxSh $ warnMsg id 3014 "== in place of = is" - bashism (TC_Binary id SingleBracket "=~" _ _) = - warnMsg id 3015 "=~ regex matching is" - bashism (T_SimpleCommand id _ [asStr -> Just "test", lhs, asStr -> Just "=~", rhs]) = - warnMsg id 3015 "=~ regex matching is" - bashism (TC_Unary id SingleBracket "-v" _) = - warnMsg id 3016 "unary -v (in place of [ -n \"${var+x}\" ]) is" - bashism (T_SimpleCommand id _ [asStr -> Just "test", asStr -> Just "-v", _]) = - warnMsg id 3016 "unary -v (in place of [ -n \"${var+x}\" ]) is" - bashism (TC_Unary id _ "-a" _) = - warnMsg id 3017 "unary -a in place of -e is" - bashism (TC_Unary id _ "-o" _) = - warnMsg id 3062 "unary -o to check options is" - bashism (T_SimpleCommand id _ [asStr -> Just "test", asStr -> Just "-a", _]) = - warnMsg id 3017 "unary -a in place of -e is" + + bashism (TC_Binary id _ op _ _) = + checkTestOp bashismBinaryTestFlags op id + bashism (T_SimpleCommand id _ [asStr -> Just "test", lhs, asStr -> Just op, rhs]) = + checkTestOp bashismBinaryTestFlags op id + bashism (TC_Unary id _ op _) = + checkTestOp bashismUnaryTestFlags op id + bashism (T_SimpleCommand id _ [asStr -> Just "test", asStr -> Just op, _]) = + checkTestOp bashismUnaryTestFlags op id + bashism (TA_Unary id op _) | op `elem` [ "|++", "|--", "++|", "--|"] = warnMsg id 3018 $ filter (/= '|') op ++ " is" @@ -529,6 +497,52 @@ checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do Assignment (_, _, name, _) -> name == var _ -> False + checkTestOp table op id = sequence_ $ do + (code, shells, msg) <- Map.lookup op table + guard . not $ shellType params `elem` shells + return $ warnMsg id code (msg op) + + +buildTestFlagMap list = Map.fromList $ concatMap (\(x,y) -> map (\c -> (c,y)) x) list +bashismBinaryTestFlags = buildTestFlagMap [ + -- ([list of applicable flags], + -- (error code, exempt shells, message builder :: String -> String)), + -- + -- Distinct error codes allow the wiki to give more helpful, targeted + -- information. + (["<", ">", "\\<", "\\>", "<=", ">=", "\\<=", "\\>="], + (3012, [Dash, BusyboxSh], \op -> "lexicographical " ++ op ++ " is")), + (["-nt", "-ot", "-ef"], + (3013, [Dash, BusyboxSh], \op -> op ++ " is")), + (["=="], + (3014, [BusyboxSh], \op -> op ++ " in place of = is")), + (["=~"], + (3015, [], \op -> op ++ " regex matching is")), + + ([], (0,[],const "")) + ] +bashismUnaryTestFlags = buildTestFlagMap [ + (["-v"], + (3016, [], \op -> "test " ++ op ++ " (in place of [ -n \"${var+x}\" ]) is")), + (["-a"], + (3017, [], \op -> "unary " ++ op ++ " in place of -e is")), + (["-o"], + (3062, [], \op -> "test " ++ op ++ " to check options is")), + (["-R"], + (3063, [], \op -> "test " ++ op ++ " and namerefs in general are")), + (["-N"], + (3064, [], \op -> "test " ++ op ++ " is")), + (["-k"], + (3065, [Dash, BusyboxSh], \op -> "test " ++ op ++ " is")), + (["-G"], + (3066, [Dash, BusyboxSh], \op -> "test " ++ op ++ " is")), + (["-O"], + (3067, [Dash, BusyboxSh], \op -> "test " ++ op ++ " is")), + + ([], (0,[],const "")) + ] + + prop_checkEchoSed1 = verify checkEchoSed "FOO=$(echo \"$cow\" | sed 's/foo/bar/g')" prop_checkEchoSed1b = verify checkEchoSed "FOO=$(sed 's/foo/bar/g' <<< \"$cow\")" prop_checkEchoSed2 = verify checkEchoSed "rm $(echo $cow | sed -e 's,foo,bar,')" From f78714e0f6070bc4efa2f7c11c1ad632e8c250a2 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Fri, 11 Apr 2025 19:14:53 -0700 Subject: [PATCH 761/763] Add ":" alongside "true" for SC2015 --- src/ShellCheck/Analytics.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 5f8c84c..ac686ff 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -890,7 +890,7 @@ checkShorthandIf params x@(T_OrIf _ (T_AndIf id _ b) (T_Pipeline _ _ t)) where isOk [t] = isAssignment t || fromMaybe False (do name <- getCommandBasename t - return $ name `elem` ["echo", "exit", "return", "printf", "true"]) + return $ name `elem` ["echo", "exit", "return", "printf", "true", ":"]) isOk _ = False inCondition = isCondition $ getPath (parentMap params) x checkShorthandIf _ _ = return () From b381658dbc74ebe7141717f7723be3a8b39121f9 Mon Sep 17 00:00:00 2001 From: Ian Ehrenwald Date: Fri, 25 Apr 2025 14:11:07 -0400 Subject: [PATCH 762/763] Add python3 to the list of badShells --- src/ShellCheck/Parser.hs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index d019d89..84c3ce4 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -3397,6 +3397,7 @@ readScriptFile sourced = do "fish", "perl", "python", + "python3", "ruby", "tcsh", "zsh" From 47d358c1d44a84a1d54a4118fba4cf7668a9225d Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 17 May 2025 00:55:50 +0000 Subject: [PATCH 763/763] Tighten SC2333/SC2334 to only trigger against literals. --- src/ShellCheck/ASTLib.hs | 6 ++++++ src/ShellCheck/Analytics.hs | 29 +++++++++++++++++++---------- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 6b26b22..1e1b9cd 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -446,6 +446,12 @@ getLiteralStringExt more = g -- Is this token a string literal? isLiteral t = isJust $ getLiteralString t +-- Is this token a string literal number? +isLiteralNumber t = fromMaybe False $ do + s <- getLiteralString t + guard $ all isDigit s + return True + -- Escape user data for messages. -- Messages generally avoid repeating user data, but sometimes it's helpful. e4m = escapeForMessage diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 073e911..2e9a3bd 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1635,8 +1635,8 @@ checkOrNeq _ (T_OrIf id lhs rhs) = sequence_ $ do checkOrNeq _ _ = return () -prop_checkAndEq1 = verify checkAndEq "if [[ $lol -eq cow && $lol -eq foo ]]; then echo foo; fi" -prop_checkAndEq2 = verify checkAndEq "(( a==lol && a==foo ))" +prop_checkAndEq1 = verifyNot checkAndEq "cow=0; foo=0; if [[ $lol -eq cow && $lol -eq foo ]]; then echo foo; fi" +prop_checkAndEq2 = verifyNot checkAndEq "lol=0 foo=0; (( a==lol && a==foo ))" prop_checkAndEq3 = verify checkAndEq "[ \"$a\" = lol && \"$a\" = foo ]" prop_checkAndEq4 = verifyNot checkAndEq "[ a = $cow && b = $foo ]" prop_checkAndEq5 = verifyNot checkAndEq "[[ $a = /home && $a = */public_html/* ]]" @@ -1644,25 +1644,34 @@ prop_checkAndEq6 = verify checkAndEq "[ $a = a ] && [ $a = b ]" prop_checkAndEq7 = verify checkAndEq "[ $a = a ] && [ $a = b ] || true" prop_checkAndEq8 = verifyNot checkAndEq "[[ $a == x && $a == x ]]" prop_checkAndEq9 = verifyNot checkAndEq "[ 0 -eq $FOO ] && [ 0 -eq $BAR ]" +prop_checkAndEq10 = verify checkAndEq "(( a == 1 && a == 2 ))" +prop_checkAndEq11 = verify checkAndEq "[ $x -eq 1 ] && [ $x -eq 2 ]" +prop_checkAndEq12 = verify checkAndEq "[ 1 -eq $x ] && [ $x -eq 2 ]" +prop_checkAndEq13 = verifyNot checkAndEq "[ 1 -eq $x ] && [ $x -eq 1 ]" +prop_checkAndEq14 = verifyNot checkAndEq "[ $a = $b ] && [ $a = $c ]" + +checkAndEqOperands "-eq" rhs1 rhs2 = isLiteralNumber rhs1 && isLiteralNumber rhs2 +checkAndEqOperands op rhs1 rhs2 | op == "=" || op == "==" = isLiteral rhs1 && isLiteral rhs2 +checkAndEqOperands _ _ _ = False -- For test-level "and": [ x = y -a x = z ] checkAndEq _ (TC_And id typ op (TC_Binary _ _ op1 lhs1 rhs1 ) (TC_Binary _ _ op2 lhs2 rhs2)) - | (op1 == op2 && (op1 == "-eq" || op1 == "=" || op1 == "==")) && lhs1 == lhs2 && rhs1 /= rhs2 && not (any isGlob [rhs1,rhs2]) = - warn id 2055 $ "You probably wanted " ++ (if typ == SingleBracket then "-o" else "||") ++ " here, otherwise it's always false." + | op1 == op2 && lhs1 == lhs2 && rhs1 /= rhs2 && checkAndEqOperands op1 rhs1 rhs2 = + warn id 2333 $ "You probably wanted " ++ (if typ == SingleBracket then "-o" else "||") ++ " here, otherwise it's always false." -- For arithmetic context "and" -checkAndEq _ (TA_Binary id "&&" (TA_Binary _ "==" word1 _) (TA_Binary _ "==" word2 _)) - | word1 == word2 = - warn id 2056 "You probably wanted || here, otherwise it's always false." +checkAndEq _ (TA_Binary id "&&" (TA_Binary _ "==" lhs1 rhs1) (TA_Binary _ "==" lhs2 rhs2)) + | lhs1 == lhs2 && isLiteralNumber rhs1 && isLiteralNumber rhs2 = + warn id 2334 "You probably wanted || here, otherwise it's always false." -- For command level "and": [ x = y ] && [ x = z ] checkAndEq _ (T_AndIf id lhs rhs) = sequence_ $ do (lhs1, op1, rhs1) <- getExpr lhs (lhs2, op2, rhs2) <- getExpr rhs - guard $ op1 == op2 && op1 `elem` ["-eq", "=", "=="] + guard $ op1 == op2 guard $ lhs1 == lhs2 && rhs1 /= rhs2 - guard . not $ any isGlob [rhs1, rhs2] - return $ warn id 2252 "You probably wanted || here, otherwise it's always false." + guard $ checkAndEqOperands op1 rhs1 rhs2 + return $ warn id 2333 "You probably wanted || here, otherwise it's always false." where getExpr x = case x of