From 9423691039f6b3ed91d61dcf94289b46ec35944b Mon Sep 17 00:00:00 2001 From: Glen Mailer Date: Mon, 12 Aug 2019 22:24:47 +0100 Subject: [PATCH 001/459] Mention the CircleCI shellcheck orb in the README. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 9237460..45b5a23 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,7 @@ Services and platforms that have ShellCheck pre-installed and ready to use: * [Codacy](https://www.codacy.com/) * [Code Climate](https://codeclimate.com/) * [Code Factor](https://www.codefactor.io/) +* [CircleCI](https://circleci.com) via the [ShellCheck Orb](https://circleci.com/orbs/registry/orb/circleci/shellcheck) Services and platforms with third party plugins: From fdd02c94c01b81b78e2dada903ea88e29a39befe Mon Sep 17 00:00:00 2001 From: Gandalf- Date: Sun, 22 Dec 2019 23:11:20 -0800 Subject: [PATCH 002/459] Issue 1759 mapfile and process substition https://github.com/koalaman/shellcheck/issues/1759 When a simple process substition is used, this tripped up the getMapfileArray function by making the last argument not a variable --- src/ShellCheck/Analytics.hs | 1 + src/ShellCheck/AnalyzerLib.hs | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 582b8cd..d6ea460 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2190,6 +2190,7 @@ prop_checkUnassignedReferences36= verifyNotTree checkUnassignedReferences "read prop_checkUnassignedReferences37= verifyNotTree checkUnassignedReferences "var=howdy; printf -v 'array[0]' %s \"$var\"; printf %s \"${array[0]}\";" prop_checkUnassignedReferences38= verifyTree (checkUnassignedReferences' True) "echo $VAR" prop_checkUnassignedReferences39= verifyNotTree checkUnassignedReferences "builtin export var=4; echo $var" +prop_checkUnassignedReferences40= verifyNotTree checkUnassignedReferences "mapfile -t files <(cat); echo \"${files[@]}\"" checkUnassignedReferences = checkUnassignedReferences' False checkUnassignedReferences' includeGlobals params t = warnings diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 590889c..1e30c7a 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -670,13 +670,17 @@ getModifiedVariableCommand base@(T_SimpleCommand id cmdPrefix (T_NormalWord _ (T -- mapfile has some curious syntax allowing flags plus 0..n variable names -- where only the first non-option one is used if any. Here we cheat and - -- just get the last one, if it's a variable name. + -- just get the last one, if it's a variable name, and omitting process + -- substitions. getMapfileArray base arguments = do - lastArg <- listToMaybe (reverse arguments) + lastArg <- listToMaybe (filter notProcSub $ reverse arguments) name <- getLiteralString lastArg guard $ isVariableName name return (base, lastArg, name, DataArray SourceExternal) + notProcSub (T_NormalWord _ (T_ProcSub{} :_)) = False + notProcSub _ = True + -- get all the array variables used in read, e.g. read -a arr getReadArrayVariables args = map (getLiteralArray . snd) From a82e606e8d18e03ff18072a064b42cc8413c0fc6 Mon Sep 17 00:00:00 2001 From: Peter Gromov Date: Fri, 31 Jan 2020 14:49:25 +0100 Subject: [PATCH 003/459] Don't trigger SC2154 (unassigned var) in `-n`/`-z` expressions #1583 --- src/ShellCheck/Analytics.hs | 10 ++++++++++ src/ShellCheck/AnalyzerLib.hs | 11 ++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 582b8cd..cf952f5 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2191,6 +2191,16 @@ prop_checkUnassignedReferences37= verifyNotTree checkUnassignedReferences "var=h prop_checkUnassignedReferences38= verifyTree (checkUnassignedReferences' True) "echo $VAR" prop_checkUnassignedReferences39= verifyNotTree checkUnassignedReferences "builtin export var=4; echo $var" +prop_checkUnassignedReferences_minusNPlain = verifyNotTree checkUnassignedReferences "if [ -n \"$x\" ]; then echo $x; fi" +prop_checkUnassignedReferences_minusZPlain = verifyNotTree checkUnassignedReferences "if [ -z \"$x\" ]; then echo \"\"; fi" +prop_checkUnassignedReferences_minusNBraced = verifyNotTree checkUnassignedReferences "if [ -n \"${x}\" ]; then echo $x; fi" +prop_checkUnassignedReferences_minusZBraced = verifyNotTree checkUnassignedReferences "if [ -z \"${x}\" ]; then echo \"\"; fi" +prop_checkUnassignedReferences_minusNDefault = verifyNotTree checkUnassignedReferences "if [ -n \"${x:-}\" ]; then echo $x; fi" +prop_checkUnassignedReferences_minusZDefault = verifyNotTree checkUnassignedReferences "if [ -z \"${x:-}\" ]; then echo \"\"; fi" + +prop_checkUnassignedReferences_minusZInsteadOfN = verifyTree checkUnassignedReferences "if [ -z \"$x\" ]; then echo $x; fi" +prop_checkUnassignedReferences_minusZInsteadOfNBraced = verifyTree checkUnassignedReferences "if [ -z \"${x}\" ]; then echo $x; fi" + checkUnassignedReferences = checkUnassignedReferences' False checkUnassignedReferences' includeGlobals params t = warnings where diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 590889c..b5e740a 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -510,6 +510,9 @@ getModifiedVariables t = guard . not . null $ str return (t, token, str, DataString SourceChecked) + TC_Unary _ _ "-n" (T_NormalWord _ [T_DoubleQuoted _ [db@(T_DollarBraced _ _ _)]]) -> + [(t, t, getBracedReference (bracedString db), DataString SourceChecked)] + T_DollarBraced _ _ l -> maybeToList $ do let string = bracedString t let modifier = getBracedModifier string @@ -719,7 +722,9 @@ getOffsetReferences mods = fromMaybe [] $ do getReferencedVariables parents t = case t of T_DollarBraced id _ l -> let str = bracedString t in - (t, t, getBracedReference str) : + if isMinusZTest t + then [] + else (t, t, getBracedReference str) : map (\x -> (l, l, x)) ( getIndexReferences str ++ getOffsetReferences (getBracedModifier str)) @@ -774,6 +779,10 @@ getReferencedVariables parents t = this: TA_Assignment _ "=" lhs _ :_ -> lhs == t _ -> False + isMinusZTest t = case getPath parents t of + _ : T_DoubleQuoted _ [_] : T_NormalWord _ [_] : TC_Unary _ SingleBracket "-z" _ : _ -> True + _ -> False + dataTypeFrom defaultType v = (case v of T_Array {} -> DataArray; _ -> defaultType) $ SourceFrom [v] From 00574dd1fce3b453de151cb53a73075b959fc59c Mon Sep 17 00:00:00 2001 From: Luke Davis Date: Tue, 3 Mar 2020 13:23:24 -0700 Subject: [PATCH 004/459] Add conda install instructions --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 5acfb89..a8ec54b 100644 --- a/README.md +++ b/README.md @@ -197,6 +197,10 @@ Or Windows (via [scoop](http://scoop.sh)): C:\> scoop install shellcheck ``` +From [conda-forge](https://anaconda.org/conda-forge/shellcheck): + + conda install -c conda-forge shellcheck + From Snap Store: snap install --channel=edge shellcheck From 84d6e53659c44b35a198b3ec80759008ba21c481 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 4 Apr 2020 19:29:28 -0700 Subject: [PATCH 005/459] Update Changelog with new version --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e5ca19..181cc54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +## Git + ## v0.7.1 - 2020-04-04 ### Fixed - `-f diff` no longer claims that it found more issues when it didn't From a30e42ab05cd1c08cc2caddca9d14346c21bbd14 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 4 Apr 2020 19:30:13 -0700 Subject: [PATCH 006/459] Filter GitHub uploads by tag --- .github_deploy | 1 + 1 file changed, 1 insertion(+) diff --git a/.github_deploy b/.github_deploy index 8f648ed..e5d6c3e 100755 --- a/.github_deploy +++ b/.github_deploy @@ -50,6 +50,7 @@ do for file in deploy/* do [[ $file == *.@(xz|gz|zip) ]] || continue + [[ $file == *"$tag"* ]] || continue files+=(-a "$file") done hub release edit "${files[@]}" "$tag" || exit 1 From d2fa88dd91f71f561d151bf46bd3bfb93255be91 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 14:04:23 -0400 Subject: [PATCH 007/459] Simplify nameExpansion --- src/ShellCheck/AnalyzerLib.hs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 453d53d..73449d7 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -840,10 +840,9 @@ getBracedReference s = fromMaybe s $ if c `elem` "*@#?-$!" then return [c] else fail "not special" getSpecial _ = fail "empty" - nameExpansion ('!':rest) = do -- e.g. ${!foo*bar*} - let suffix = dropWhile isVariableChar rest - guard $ suffix /= rest -- e.g. ${!@} - first <- suffix !!! 0 + nameExpansion ('!':next:rest) = do -- e.g. ${!foo*bar*} + guard $ isVariableChar next -- e.g. ${!@} + first <- find (not . isVariableChar) rest guard $ first `elem` "*?" return "" nameExpansion _ = Nothing From 01f442346573361920b5484c0e597716b5e97b06 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 5 Apr 2020 11:38:22 -0700 Subject: [PATCH 008/459] Disable SC2257 about > $((i=42)) for Dash --- src/ShellCheck/Analytics.hs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index a0f5088..f5a8047 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -3596,7 +3596,8 @@ prop_checkModifiedArithmeticInRedirection2 = verify checkModifiedArithmeticInRed prop_checkModifiedArithmeticInRedirection3 = verifyNot checkModifiedArithmeticInRedirection "while true; do true; done > $((i++))" prop_checkModifiedArithmeticInRedirection4 = verify checkModifiedArithmeticInRedirection "cat <<< $((i++))" prop_checkModifiedArithmeticInRedirection5 = verify checkModifiedArithmeticInRedirection "cat << foo\n$((i++))\nfoo\n" -checkModifiedArithmeticInRedirection _ t = +prop_checkModifiedArithmeticInRedirection6 = verifyNot checkModifiedArithmeticInRedirection "#!/bin/dash\nls > $((i=i+1))" +checkModifiedArithmeticInRedirection params t = unless (shellType params == Dash) $ case t of T_Redirecting _ redirs (T_SimpleCommand _ _ (_:_)) -> mapM_ checkRedirs redirs _ -> return () From 2a8170ba05c35b2a31950b2c380e0ad6247121c9 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 15:01:57 -0400 Subject: [PATCH 009/459] Use force instead of reimplementing it --- src/ShellCheck/AnalyzerLib.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 453d53d..086bd7c 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -178,7 +178,7 @@ makeCommentWithFix severity id code str fix = withFix = comment { tcFix = Just fix } - in withFix `deepseq` withFix + in force withFix makeParameters spec = let params = Parameters { From b0dbc79f6973917aa51f1b9629e7a51a03162751 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 15:07:36 -0400 Subject: [PATCH 010/459] Remove unnecessary Maybe from isQuoteFreeElement --- src/ShellCheck/AnalyzerLib.hs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 086bd7c..0a4145a 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -293,15 +293,15 @@ isQuoteFree = isQuoteFreeNode False isQuoteFreeNode strict tree t = - (isQuoteFreeElement t == Just True) || + isQuoteFreeElement t || headOrDefault False (mapMaybe isQuoteFreeContext (drop 1 $ getPath tree t)) where -- Is this node self-quoting in itself? isQuoteFreeElement t = case t of - T_Assignment {} -> return True - T_FdRedirect {} -> return True - _ -> Nothing + T_Assignment {} -> True + T_FdRedirect {} -> True + _ -> False -- Are any subnodes inherently self-quoting? isQuoteFreeContext t = From b3c04ce3d0e0e95a0b3d9be9c58c18426d899d22 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 15:21:44 -0400 Subject: [PATCH 011/459] Implement findFirst in terms of foldr --- src/ShellCheck/AnalyzerLib.hs | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 0a4145a..4348438 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -388,14 +388,13 @@ parents params = getPath (parentMap params) -- Find the first match in a list where the predicate is Just True. -- Stops if it's Just False and ignores Nothing. findFirst :: (a -> Maybe Bool) -> [a] -> Maybe a -findFirst p l = - case l of - [] -> Nothing - (x:xs) -> - case p x of - Just True -> return x - Just False -> Nothing - Nothing -> findFirst p xs +findFirst p = foldr go Nothing + where + go x acc = + case p x of + Just True -> return x + Just False -> Nothing + Nothing -> acc -- Check whether a word is entirely output from a single command tokenIsJustCommandOutput t = case t of From 14ee462ccd572c14ca292b47330e87c9307e0a7c Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 15:27:11 -0400 Subject: [PATCH 012/459] Use execState instead of reimplementing it --- src/ShellCheck/AnalyzerLib.hs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 4348438..8f168dc 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -409,8 +409,7 @@ tokenIsJustCommandOutput t = case t of -- TODO: Replace this with a proper Control Flow Graph getVariableFlow params t = - let (_, stack) = runState (doStackAnalysis startScope endScope t) [] - in reverse stack + reverse $ execState (doStackAnalysis startScope endScope t) [] where startScope t = let scopeType = leadType params t From f55d8c45e567398cfd9effcf08ad6d5706936268 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 15:30:28 -0400 Subject: [PATCH 013/459] Simplify causesSubshell --- src/ShellCheck/AnalyzerLib.hs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 8f168dc..fd0f46d 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -460,11 +460,9 @@ leadType params t = causesSubshell = do (T_Pipeline _ _ list) <- parentPipeline - if length list <= 1 - then return False - else if not $ hasLastpipe params - then return True - else return . not $ (getId . head $ reverse list) == getId t + return $ case list of + _:_:_ -> not (hasLastpipe params) || getId (last list) /= getId t + _ -> False getModifiedVariables t = case t of From f833ee3d5ada7cf4d648fb16ef143e09550b9fac Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 15:35:09 -0400 Subject: [PATCH 014/459] Use a list comprehension instead of a concatMap with extra lists --- src/ShellCheck/AnalyzerLib.hs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index fd0f46d..31ee431 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -467,11 +467,7 @@ leadType params t = getModifiedVariables t = case t of T_SimpleCommand _ vars [] -> - concatMap (\x -> case x of - T_Assignment id _ name _ w -> - [(x, x, name, dataTypeFrom DataString w)] - _ -> [] - ) vars + [(x, x, name, dataTypeFrom DataString w) | x@(T_Assignment id _ name _ w) <- vars] c@T_SimpleCommand {} -> getModifiedVariableCommand c From 67e091674ed35cb20b9053e99e4d38e3d68286e8 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 15:36:10 -0400 Subject: [PATCH 015/459] Remove unnecessary maybeToList The functions we use here are polymorphic enough to work in the [] monad, so there's no point to use them in the Maybe monad and then convert. --- src/ShellCheck/AnalyzerLib.hs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 31ee431..188054c 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -475,7 +475,7 @@ getModifiedVariables t = [(t, v, name, DataString $ SourceFrom [v])] TA_Unary _ "|++" v@(TA_Variable _ name _) -> [(t, v, name, DataString $ SourceFrom [v])] - TA_Assignment _ op (TA_Variable _ name _) rhs -> maybeToList $ do + TA_Assignment _ op (TA_Variable _ name _) rhs -> do guard $ op `elem` ["=", "*=", "/=", "%=", "+=", "-=", "<<=", ">>=", "&=", "^=", "|="] return (t, t, name, DataString $ SourceFrom [rhs]) @@ -487,17 +487,17 @@ getModifiedVariables t = -- Count [[ -v foo ]] as an "assignment". -- This is to prevent [ -v foo ] being unassigned or unused. - TC_Unary id _ "-v" token -> maybeToList $ do + TC_Unary id _ "-v" token -> do str <- fmap (takeWhile (/= '[')) $ -- Quoted index flip getLiteralStringExt token $ \x -> case x of T_Glob _ s -> return s -- Unquoted index - _ -> Nothing + _ -> [] guard . not . null $ str return (t, token, str, DataString SourceChecked) - T_DollarBraced _ _ l -> maybeToList $ do + T_DollarBraced _ _ l -> do let string = bracedString t let modifier = getBracedModifier string guard $ any (`isPrefixOf` modifier) ["=", ":="] @@ -747,9 +747,9 @@ getReferencedVariables parents t = literalizer t = case t of T_Glob _ s -> return s -- Also when parsed as globs - _ -> Nothing + _ -> [] - getIfReference context token = maybeToList $ do + getIfReference context token = do str@(h:_) <- getLiteralStringExt literalizer token when (isDigit h) $ fail "is a number" return (context, token, getBracedReference str) From f109f9ab9270dd78bd8f8dc2158c9ce4904f599e Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 15:38:16 -0400 Subject: [PATCH 016/459] Remove unnecessary as-patterns --- src/ShellCheck/AnalyzerLib.hs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 188054c..543f6e6 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -468,8 +468,8 @@ getModifiedVariables t = case t of T_SimpleCommand _ vars [] -> [(x, x, name, dataTypeFrom DataString w) | x@(T_Assignment id _ name _ w) <- vars] - c@T_SimpleCommand {} -> - getModifiedVariableCommand c + T_SimpleCommand {} -> + getModifiedVariableCommand t TA_Unary _ "++|" v@(TA_Variable _ name _) -> [(t, v, name, DataString $ SourceFrom [v])] @@ -503,10 +503,10 @@ getModifiedVariables t = guard $ any (`isPrefixOf` modifier) ["=", ":="] return (t, t, getBracedReference string, DataString $ SourceFrom [l]) - t@(T_FdRedirect _ ('{':var) op) -> -- {foo}>&2 modifies foo + T_FdRedirect _ ('{':var) op -> -- {foo}>&2 modifies foo [(t, t, takeWhile (/= '}') var, DataString SourceInteger) | not $ isClosingFileOp op] - t@(T_CoProc _ name _) -> + T_CoProc _ name _ -> [(t, t, fromMaybe "COPROC" name, DataArray SourceInteger)] --Points to 'for' rather than variable @@ -730,7 +730,7 @@ getReferencedVariables parents t = (t, t, "output") ] - t@(T_FdRedirect _ ('{':var) op) -> -- {foo}>&- references and closes foo + T_FdRedirect _ ('{':var) op -> -- {foo}>&- references and closes foo [(t, t, takeWhile (/= '}') var) | isClosingFileOp op] x -> getReferencedVariableCommand x where From e4eb2d157f8211e14b66d0718fa4d90e3fefbd8a Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 16:03:49 -0400 Subject: [PATCH 017/459] Remove an unnecessary operator section --- src/ShellCheck/AnalyzerLib.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 543f6e6..4cd4b5e 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -564,7 +564,7 @@ getModifiedVariableCommand base@(T_SimpleCommand id cmdPrefix (T_NormalWord _ (T let params = map getLiteral rest readArrayVars = getReadArrayVariables rest in - catMaybes . (++ readArrayVars) . takeWhile isJust . reverse $ params + catMaybes $ takeWhile isJust (reverse params) ++ readArrayVars "getopts" -> case rest of opts:var:_ -> maybeToList $ getLiteral var From 2ebf522a52efe4ef092009f0db6da4cdd25edf5b Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 16:07:19 -0400 Subject: [PATCH 018/459] Simplify isArrayFlag --- src/ShellCheck/AnalyzerLib.hs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 4cd4b5e..8208a5b 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -669,12 +669,10 @@ getModifiedVariableCommand base@(T_SimpleCommand id cmdPrefix (T_NormalWord _ (T map (getLiteralArray . snd) (filter (isArrayFlag . fst) (zip args (tail args))) - isArrayFlag x = fromMaybe False $ do - str <- getLiteralString x - return $ case str of - '-':'-':_ -> False - '-':str -> 'a' `elem` str - _ -> False + isArrayFlag x = case getLiteralString x of + Just ('-':'-':_) -> False + Just ('-':str) -> 'a' `elem` str + _ -> False -- get the FLAGS_ variable created by a shflags DEFINE_ call getFlagVariable (n:v:_) = do From 4604066c3729766b49da0251786286b6c1ae8caa Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 16:16:12 -0400 Subject: [PATCH 019/459] Use head instead of (!! 0) --- src/ShellCheck/AnalyzerLib.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 8208a5b..c5eaea6 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -798,7 +798,7 @@ getVariablesFromLiteralToken token = prop_getVariablesFromLiteral1 = getVariablesFromLiteral "$foo${bar//a/b}$BAZ" == ["foo", "bar", "BAZ"] getVariablesFromLiteral string = - map (!! 0) $ matchAllSubgroups variableRegex string + map head $ matchAllSubgroups variableRegex string where variableRegex = mkRegex "\\$\\{?([A-Za-z0-9_]+)" From 1cf0aa25e9a76585f34798628cf7cc7d35642970 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 16:19:18 -0400 Subject: [PATCH 020/459] Simplify dropPrefix --- src/ShellCheck/AnalyzerLib.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index c5eaea6..26e65e6 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -820,8 +820,8 @@ getBracedReference s = fromMaybe s $ nameExpansion s `mplus` takeName noPrefix `mplus` getSpecial noPrefix `mplus` getSpecial s where noPrefix = dropPrefix s - dropPrefix (c:rest) = if c `elem` "!#" then rest else c:rest - dropPrefix "" = "" + dropPrefix (c:rest) | c `elem` "!#" = rest + dropPrefix cs = cs takeName s = do let name = takeWhile isVariableChar s guard . not $ null name From ca41440a678879d586320250a4b91913d1256a6e Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 16:21:07 -0400 Subject: [PATCH 021/459] Simplify getSpecial --- src/ShellCheck/AnalyzerLib.hs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 26e65e6..95b236a 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -826,9 +826,8 @@ getBracedReference s = fromMaybe s $ let name = takeWhile isVariableChar s guard . not $ null name return name - getSpecial (c:_) = - if c `elem` "*@#?-$!" then return [c] else fail "not special" - getSpecial _ = fail "empty" + getSpecial (c:_) | c `elem` "*@#?-$!" = return [c] + getSpecial _ = fail "empty or not special" nameExpansion ('!':rest) = do -- e.g. ${!foo*bar*} let suffix = dropWhile isVariableChar rest From 0cc5ed4563ccd87b200dd41c2801f8d4d79b8f15 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 16:25:43 -0400 Subject: [PATCH 022/459] Don't bother with asks if you're just immediately binding the result anyway --- src/ShellCheck/AnalyzerLib.hs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 95b236a..32de46a 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -354,8 +354,8 @@ getClosestCommand tree t = -- Like above, if koala_man knew Haskell when starting this project. getClosestCommandM t = do - tree <- asks parentMap - return $ getClosestCommand tree t + params <- ask + return $ getClosestCommand (parentMap params) t -- Is the token used as a command name (the first word in a T_SimpleCommand)? usedAsCommandName tree token = go (getId token) (tail $ getPath tree token) @@ -377,8 +377,8 @@ getPath tree t = t : -- Version of the above taking the map from the current context -- Todo: give this the name "getPath" getPathM t = do - map <- asks parentMap - return $ getPath map t + params <- ask + return $ getPath (parentMap params) t isParentOf tree parent child = elem (getId parent) . map getId $ getPath tree child @@ -866,8 +866,8 @@ headOrDefault def _ = def -- Run a command if the shell is in the given list whenShell l c = do - shell <- asks shellType - when (shell `elem` l ) c + params <- ask + when (shellType params `elem` l ) c filterByAnnotation asSpec params = From fb55072302c14d4785875357d4400de736f4eeb9 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 16:30:59 -0400 Subject: [PATCH 023/459] Implement supportsArrays with pattern-matching --- src/ShellCheck/AnalyzerLib.hs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 32de46a..a3fa29d 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -934,7 +934,9 @@ getOpts string flags = process flags more <- process rest2 return $ (flag1, token1) : more -supportsArrays shell = shell == Bash || shell == Ksh +supportsArrays Bash = True +supportsArrays Ksh = True +supportsArrays _ = False -- Returns true if the shell is Bash or Ksh (sorry for the name, Ksh) isBashLike :: Parameters -> Bool From d22e0aa4a79ddd8182c81898b790c9b9bb850937 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 16:38:52 -0400 Subject: [PATCH 024/459] Simplify process Note to self: This is a lot like foldr or traverse, and would be trivial to implement as such if it didn't need to peek ahead when takesArg is true. I wonder if there's a clean way to implement it in terms of one of them anyway. --- src/ShellCheck/AnalyzerLib.hs | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index a3fa29d..7c2440b 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -919,20 +919,15 @@ getOpts string flags = process flags flagMap = Map.fromList $ ("", False) : flagList string process [] = return [] - process [(token, flag)] = do + process ((token1, flag):rest1) = do takesArg <- Map.lookup flag flagMap - guard $ not takesArg - return [(flag, token)] - process ((token1, flag1):rest2@((token2, flag2):rest)) = do - takesArg <- Map.lookup flag1 flagMap - if takesArg - then do - guard $ null flag2 - more <- process rest - return $ (flag1, token2) : more - else do - more <- process rest2 - return $ (flag1, token1) : more + (token, rest) <- if takesArg + then case rest1 of + (token2, ""):rest2 -> return (token2, rest2) + _ -> fail "takesArg without valid arg" + else return (token1, rest1) + more <- process rest + return $ (flag, token) : more supportsArrays Bash = True supportsArrays Ksh = True From 8f105074feebda422e4242d04bb4afdf1e0431cf Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 19:01:56 -0400 Subject: [PATCH 025/459] Simplify getCommandNameAndToken --- src/ShellCheck/ASTLib.hs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 2b40705..05c3838 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -317,13 +317,11 @@ getCommandNameAndToken :: Token -> (Maybe String, Token) getCommandNameAndToken t = fromMaybe (Nothing, t) $ do (T_SimpleCommand _ _ (w:rest)) <- getCommand t s <- getLiteralString w - if "busybox" `isSuffixOf` s || "builtin" == s - then - case rest of - (applet:_) -> return (getLiteralString applet, applet) - _ -> return (Just s, w) - else - return (Just s, w) + return $ case rest of + (applet:_) | "busybox" `isSuffixOf` s || "builtin" == s -> + (getLiteralString applet, applet) + _ -> + (Just s, w) -- If a command substitution is a single command, get its name. From b6cff5ea0e1626ad3251ffa018b5990d3b7c23c5 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 19:06:16 -0400 Subject: [PATCH 026/459] Simplify getAssociativeArrays --- src/ShellCheck/ASTLib.hs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 05c3838..9494150 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -398,10 +398,10 @@ getAssociativeArrays t = f t@T_SimpleCommand {} = sequence_ $ do name <- getCommandName t let assocNames = ["declare","local","typeset"] - guard $ elem name assocNames + guard $ name `elem` assocNames let flags = getAllFlags t - guard $ elem "A" $ map snd flags - let args = map fst . filter ((==) "" . snd) $ flags + guard $ "A" `elem` map snd flags + let args = [arg | (arg, "") <- flags] let names = mapMaybe (getLiteralStringExt nameAssignments) args return $ tell names f _ = return () From 322842b57e370d484355172497c69bd163184ef4 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 19:29:40 -0400 Subject: [PATCH 027/459] Remove unnecessary monadicity from wordToPseudoGlob --- src/ShellCheck/ASTLib.hs | 27 +++++++++++++-------------- src/ShellCheck/Analytics.hs | 16 +++++----------- 2 files changed, 18 insertions(+), 25 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 9494150..758d79c 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -419,25 +419,25 @@ data PseudoGlob = PGAny | PGMany | PGChar Char -- Turn a word into a PG pattern, replacing all unknown/runtime values with -- PGMany. -wordToPseudoGlob :: Token -> Maybe [PseudoGlob] +wordToPseudoGlob :: Token -> [PseudoGlob] wordToPseudoGlob word = - simplifyPseudoGlob . concat <$> mapM f (getWordParts word) + simplifyPseudoGlob . concatMap f $ getWordParts word where f x = case x of - T_Literal _ s -> return $ map PGChar s - T_SingleQuoted _ s -> return $ map PGChar s + T_Literal _ s -> map PGChar s + T_SingleQuoted _ s -> map PGChar s - T_DollarBraced {} -> return [PGMany] - T_DollarExpansion {} -> return [PGMany] - T_Backticked {} -> return [PGMany] + T_DollarBraced {} -> [PGMany] + T_DollarExpansion {} -> [PGMany] + T_Backticked {} -> [PGMany] - T_Glob _ "?" -> return [PGAny] - T_Glob _ ('[':_) -> return [PGAny] - T_Glob {} -> return [PGMany] + T_Glob _ "?" -> [PGAny] + T_Glob _ ('[':_) -> [PGAny] + T_Glob {} -> [PGMany] - T_Extglob {} -> return [PGMany] + T_Extglob {} -> [PGMany] - _ -> return [PGMany] + _ -> [PGMany] -- Turn a word into a PG pattern, but only if we can preserve -- exact semantics. @@ -500,8 +500,7 @@ pseudoGlobIsSuperSetof = matchable matchable (PGMany : rest) [] = matchable rest [] matchable _ _ = False -wordsCanBeEqual x y = fromMaybe True $ - liftM2 pseudoGlobsCanOverlap (wordToPseudoGlob x) (wordToPseudoGlob y) +wordsCanBeEqual x y = pseudoGlobsCanOverlap (wordToPseudoGlob x) (wordToPseudoGlob y) -- Is this an expansion that can be quoted, -- e.g. $(foo) `foo` $foo (but not {foo,})? diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index f5a8047..599e257 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -3137,9 +3137,7 @@ checkUnmatchableCases params t = if isConstant word then warn (getId word) 2194 "This word is constant. Did you forget the $ on a variable?" - else sequence_ $ do - pg <- wordToPseudoGlob word - return $ mapM_ (check pg) allpatterns + else mapM_ (check $ wordToPseudoGlob word) allpatterns let exactGlobs = tupMap wordToExactPseudoGlob breakpatterns let fuzzyGlobs = tupMap wordToPseudoGlob breakpatterns @@ -3152,15 +3150,13 @@ checkUnmatchableCases params t = fst3 (x,_,_) = x snd3 (_,x,_) = x tp = tokenPositions params - check target candidate = sequence_ $ do - candidateGlob <- wordToPseudoGlob candidate - guard . not $ pseudoGlobsCanOverlap target candidateGlob - return $ warn (getId candidate) 2195 - "This pattern will never match the case statement's word. Double check them." + check target candidate = unless (pseudoGlobsCanOverlap target $ wordToPseudoGlob candidate) $ + warn (getId candidate) 2195 + "This pattern will never match the case statement's word. Double check them." tupMap f l = map (\x -> (x, f x)) l checkDoms ((glob, Just x), rest) = - forM_ (find (\(_, p) -> x `pseudoGlobIsSuperSetof` p) valids) $ + forM_ (find (\(_, p) -> x `pseudoGlobIsSuperSetof` p) rest) $ \(first,_) -> do warn (getId glob) 2221 $ "This pattern always overrides a later one" <> patternContext (getId first) warn (getId first) 2222 $ "This pattern never matches because of a previous pattern" <> patternContext (getId glob) @@ -3170,8 +3166,6 @@ checkUnmatchableCases params t = case posLine . fst <$> Map.lookup id tp of Just l -> " on line " <> show l <> "." _ -> "." - - valids = [(x,y) | (x, Just y) <- rest] checkDoms _ = return () From 0f9b0f18a435cafc7cb30451c85ee6efbef029ec Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 19:30:21 -0400 Subject: [PATCH 028/459] Remove unnecessary cases from wordToPseudoGlob --- src/ShellCheck/ASTLib.hs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 758d79c..da91d09 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -427,15 +427,8 @@ wordToPseudoGlob word = T_Literal _ s -> map PGChar s T_SingleQuoted _ s -> map PGChar s - T_DollarBraced {} -> [PGMany] - T_DollarExpansion {} -> [PGMany] - T_Backticked {} -> [PGMany] - T_Glob _ "?" -> [PGAny] T_Glob _ ('[':_) -> [PGAny] - T_Glob {} -> [PGMany] - - T_Extglob {} -> [PGMany] _ -> [PGMany] From d45ab327b0ca4fe949c13b95fc7eebb8c8d74a5d Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 19:45:28 -0400 Subject: [PATCH 029/459] Only perform the comparisons once --- src/ShellCheck/Fixer.hs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/ShellCheck/Fixer.hs b/src/ShellCheck/Fixer.hs index 16bdd98..1409b24 100644 --- a/src/ShellCheck/Fixer.hs +++ b/src/ShellCheck/Fixer.hs @@ -273,10 +273,10 @@ getPrefixSum = f 0 where f sum _ PSLeaf = sum f sum target (PSBranch pivot left right cumulative) = - case () of - _ | target < pivot -> f sum target left - _ | target > pivot -> f (sum+cumulative) target right - _ -> sum+cumulative + case target `compare` pivot of + LT -> f sum target left + GT -> f (sum+cumulative) target right + EQ -> sum+cumulative -- Add a value to the Prefix Sum tree at the given index. -- Values accumulate: addPSValue 42 2 . addPSValue 42 3 == addPSValue 42 5 @@ -285,10 +285,10 @@ addPSValue key value tree = if value == 0 then tree else f tree where f PSLeaf = PSBranch key PSLeaf PSLeaf value f (PSBranch pivot left right sum) = - case () of - _ | key < pivot -> PSBranch pivot (f left) right (sum + value) - _ | key > pivot -> PSBranch pivot left (f right) sum - _ -> PSBranch pivot left right (sum + value) + case key `compare` pivot of + LT -> PSBranch pivot (f left) right (sum + value) + GT -> PSBranch pivot left (f right) sum + EQ -> PSBranch pivot left right (sum + value) prop_pstreeSumsCorrectly kvs targets = let From 773e98868daf2138b249d805e2297feeda32827a Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 19:53:40 -0400 Subject: [PATCH 030/459] Use foldr in checkFindNameGlob --- src/ShellCheck/Checks/Commands.hs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index e321102..e0a2749 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -199,12 +199,11 @@ prop_checkFindNameGlob3 = verifyNot checkFindNameGlob "find * -name '*.php'" checkFindNameGlob = CommandCheck (Basename "find") (f . arguments) where acceptsGlob s = s `elem` [ "-ilname", "-iname", "-ipath", "-iregex", "-iwholename", "-lname", "-name", "-path", "-regex", "-wholename" ] f [] = return () - f (x:xs) = g x xs - g _ [] = return () - g a (b:r) = do + f (x:xs) = foldr g (const $ return ()) xs x + g b acc a = do forM_ (getLiteralString a) $ \s -> when (acceptsGlob s && isGlob b) $ warn (getId b) 2061 $ "Quote the parameter to " ++ s ++ " so the shell won't interpret it." - g b r + acc b prop_checkNeedlessExpr = verify checkNeedlessExpr "foo=$(expr 3 + 2)" From 9027a9239fafe40463dba495bd25fdd739a4dd52 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 20:03:17 -0400 Subject: [PATCH 031/459] Use pattern matching instead of snd --- src/ShellCheck/Checks/Commands.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index e0a2749..0d887dd 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -461,8 +461,8 @@ checkMkdirDashPM = CommandCheck (Basename "mkdir") check where check t = sequence_ $ do let flags = getAllFlags t - dashP <- find ((\f -> f == "p" || f == "parents") . snd) flags - dashM <- find ((\f -> f == "m" || f == "mode") . snd) flags + dashP <- find (\(_,f) -> f == "p" || f == "parents") flags + dashM <- find (\(_,f) -> f == "m" || f == "mode") flags -- mkdir -pm 0700 dir is fine, so is ../dir, but dir/subdir is not. guard $ any couldHaveSubdirs (drop 1 $ arguments t) return $ warn (getId $ fst dashM) 2174 "When used with -p, -m only applies to the deepest directory." From e8501151dd5d4ad90687deefa434f424701b6c7c Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 20:04:54 -0400 Subject: [PATCH 032/459] Use a guard instead of unless --- src/ShellCheck/Checks/Commands.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 0d887dd..9e19ab5 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -482,7 +482,7 @@ prop_checkNonportableSignals7 = verifyNot checkNonportableSignals "trap 'stop' i checkNonportableSignals = CommandCheck (Exactly "trap") (f . arguments) where f args = case args of - first:rest -> unless (isFlag first) $ mapM_ check rest + first:rest | not $ isFlag first -> mapM_ check rest _ -> return () check param = sequence_ $ do From fa841cb2701863b84ace7d652ff977db315e599c Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 20:08:02 -0400 Subject: [PATCH 033/459] Prefer pattern matching in undirected --- src/ShellCheck/Checks/Commands.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 9e19ab5..19884ee 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -519,9 +519,9 @@ checkInteractiveSu = CommandCheck (Basename "su") f info (getId cmd) 2117 "To run commands as another user, use su -c or sudo." - undirected (T_Pipeline _ _ l) = length l <= 1 + undirected (T_Pipeline _ _ (_:_:_)) = False -- This should really just be modifications to stdin, but meh - undirected (T_Redirecting _ list _) = null list + undirected (T_Redirecting _ (_:_) _) = False undirected _ = True From 9747b1d5c3961fbcadf657dffe324b66dc5856a3 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 20:10:56 -0400 Subject: [PATCH 034/459] Simplify checkArg --- src/ShellCheck/Checks/Commands.hs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 19884ee..1633dc6 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -538,9 +538,8 @@ checkSshCommandString = CommandCheck (Basename "ssh") (f . arguments) ([], hostport:r@(_:_)) -> checkArg $ last r _ -> return () checkArg (T_NormalWord _ [T_DoubleQuoted id parts]) = - case filter (not . isConstant) parts of - [] -> return () - (x:_) -> info (getId x) 2029 + forM_ (find (not . isConstant) parts) $ + \x -> info (getId x) 2029 "Note that, unescaped, this expands on the client side." checkArg _ = return () From df4928f4e3191e18abbaa5085c13c790eeb694fe Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 20:14:03 -0400 Subject: [PATCH 035/459] Use MultiWayIf instead of case-matching on () --- src/ShellCheck/Checks/Commands.hs | 32 +++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 1633dc6..440b74a 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -19,6 +19,7 @@ -} {-# LANGUAGE TemplateHaskell #-} {-# LANGUAGE FlexibleContexts #-} +{-# LANGUAGE MultiWayIf #-} -- This module contains checks that examine specific commands by name. module ShellCheck.Checks.Commands (checker, optionalChecks, ShellCheck.Checks.Commands.runTests) where @@ -578,22 +579,21 @@ checkPrintfVar = CommandCheck (Exactly "printf") (f . arguments) where let formatCount = length formats let argCount = length more - return $ - case () of - () | argCount == 0 && formatCount == 0 -> - return () -- This is fine - () | formatCount == 0 && argCount > 0 -> - err (getId format) 2182 - "This printf format string has no variables. Other arguments are ignored." - () | any mayBecomeMultipleArgs more -> - return () -- We don't know so trust the user - () | argCount < formatCount && onlyTrailingTs formats argCount -> - return () -- Allow trailing %()Ts since they use the current time - () | argCount > 0 && argCount `mod` formatCount == 0 -> - return () -- Great: a suitable number of arguments - () -> - warn (getId format) 2183 $ - "This format string has " ++ show formatCount ++ " variables, but is passed " ++ show argCount ++ " arguments." + return $ if + | argCount == 0 && formatCount == 0 -> + return () -- This is fine + | formatCount == 0 && argCount > 0 -> + err (getId format) 2182 + "This printf format string has no variables. Other arguments are ignored." + | any mayBecomeMultipleArgs more -> + return () -- We don't know so trust the user + | argCount < formatCount && onlyTrailingTs formats argCount -> + return () -- Allow trailing %()Ts since they use the current time + | argCount > 0 && argCount `mod` formatCount == 0 -> + return () -- Great: a suitable number of arguments + | otherwise -> + warn (getId format) 2183 $ + "This format string has " ++ show formatCount ++ " variables, but is passed " ++ show argCount ++ " arguments." unless ('%' `elem` concat (oversimplify format) || isLiteral format) $ info (getId format) 2059 From cfa2a663afd83794edacd81ffc0ef9f972e5be26 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 20:21:14 -0400 Subject: [PATCH 036/459] Simplify checkSetAssignment --- src/ShellCheck/Checks/Commands.hs | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 440b74a..4f92fa9 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -661,17 +661,12 @@ prop_checkSetAssignment5 = verifyNot checkSetAssignment "set 'a=5'" prop_checkSetAssignment6 = verifyNot checkSetAssignment "set" checkSetAssignment = CommandCheck (Exactly "set") (f . arguments) where - f (var:value:rest) = - let str = literal var in - when (isVariableName str || isAssignment str) $ - msg (getId var) - f (var:_) = - when (isAssignment $ literal var) $ - msg (getId var) + f (var:rest) + | (not (null rest) && isVariableName str) || isAssignment str = + warn (getId var) 2121 "To assign a variable, use just 'var=value', no 'set ..'." + where str = literal var f _ = return () - msg id = warn id 2121 "To assign a variable, use just 'var=value', no 'set ..'." - isAssignment str = '=' `elem` str literal (T_NormalWord _ l) = concatMap literal l literal (T_Literal _ str) = str From ed331b816b083d8598de3e4ea77a27f2f2daa83a Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 20:32:39 -0400 Subject: [PATCH 037/459] Simplify warnRedundant --- src/ShellCheck/Checks/Commands.hs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 4f92fa9..c68bdd1 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -878,10 +878,10 @@ checkWhileGetoptsCase = CommandCheck (Exactly "getopts") f warnUnhandled optId caseId str = warn caseId 2213 $ "getopts specified -" ++ str ++ ", but it's not handled by this 'case'." - warnRedundant (key, expr) = sequence_ $ do - str <- key - guard $ str `notElem` ["*", ":", "?"] - return $ warn (getId expr) 2214 "This case is not specified by getopts." + warnRedundant (Just str, expr) + | str `notElem` ["*", ":", "?"] = + warn (getId expr) 2214 "This case is not specified by getopts." + warnRedundant _ = return () getHandledStrings (_, globs, _) = map (\x -> (literal x, x)) globs From 5084ba8d7ed017551de498365861f8a585eca360 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 21:23:34 -0400 Subject: [PATCH 038/459] Make skipRepeating lazier and faster --- src/ShellCheck/Checks/Commands.hs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index e321102..14a8c09 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -966,9 +966,9 @@ checkCatastrophicRm = CommandCheck (Basename "rm") $ \t -> f _ = return "" stripTrailing c = reverse . dropWhile (== c) . reverse - skipRepeating c (a:b:rest) | a == b && b == c = skipRepeating c (b:rest) - skipRepeating c (a:r) = a:skipRepeating c r - skipRepeating _ [] = [] + skipRepeating c = foldr go [] + where + go a r = a : if a == c then case r of b:rest | b == c -> rest; _ -> r else r paths = [ "", "/bin", "/etc", "/home", "/mnt", "/usr", "/usr/share", "/usr/local", From cd38afce26d9914085ea632d3eebded5fd667778 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 21:45:51 -0400 Subject: [PATCH 039/459] Make it slightly lazier still (and more clear) --- src/ShellCheck/Checks/Commands.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 14a8c09..ae02cd8 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -968,7 +968,7 @@ checkCatastrophicRm = CommandCheck (Basename "rm") $ \t -> stripTrailing c = reverse . dropWhile (== c) . reverse skipRepeating c = foldr go [] where - go a r = a : if a == c then case r of b:rest | b == c -> rest; _ -> r else r + go a r = a : case r of b:rest | b == c && a == b -> rest; _ -> r paths = [ "", "/bin", "/etc", "/home", "/mnt", "/usr", "/usr/share", "/usr/local", From facf0d1e27185861f7ede9ac1bd9803b440d6c77 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 21:59:27 -0400 Subject: [PATCH 040/459] Write getLiteralArgs with foldr and without fromMaybe or monads --- src/ShellCheck/Checks/ShellSupport.hs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 49d3212..de5f78c 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -289,10 +289,11 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do -- Get the literal options from a list of arguments, -- up until the first non-literal one getLiteralArgs :: [Token] -> [(Id, String)] - getLiteralArgs (first:rest) = fromMaybe [] $ do - str <- getLiteralString first - return $ (getId first, str) : getLiteralArgs rest - getLiteralArgs [] = [] + getLiteralArgs = foldr go [] + where + go first rest = case getLiteralString first of + Just str -> (getId first, str) : rest + Nothing -> [] -- Check a flag-option pair (such as -o errexit) checkOptions (flag@(fid,flag') : opt@(oid,opt') : rest) From 8a6679fd8a12e365411d97918206f8dc1db5c1ec Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 22:03:50 -0400 Subject: [PATCH 041/459] Remove unnecessary fromMaybe and when from bashism --- src/ShellCheck/Checks/ShellSupport.hs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index de5f78c..054661a 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -391,11 +391,10 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do ("unset", Just ["f", "v"]), ("wait", Just []) ] - bashism t@(T_SourceCommand id src _) = - let name = fromMaybe "" $ getCommandName src - in when (name == "source") $ warnMsg id "'source' in place of '.' is" - bashism (TA_Expansion _ (T_Literal id str : _)) | str `matches` radix = - when (str `matches` radix) $ warnMsg id "arithmetic base conversion is" + bashism t@(T_SourceCommand id src _) + | getCommandName src == Just "source" = warnMsg id "'source' in place of '.' is" + bashism (TA_Expansion _ (T_Literal id str : _)) + | str `matches` radix = warnMsg id "arithmetic base conversion is" where radix = mkRegex "^[0-9]+#" bashism _ = return () From 64c31d91422e3c48cc7ed0176b069fa11456cee8 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 22:13:39 -0400 Subject: [PATCH 042/459] Use fromRight instead of reimplementing it --- src/ShellCheck/Formatter/GCC.hs | 3 ++- src/ShellCheck/Formatter/JSON1.hs | 3 ++- src/ShellCheck/Formatter/TTY.hs | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/ShellCheck/Formatter/GCC.hs b/src/ShellCheck/Formatter/GCC.hs index 9c5fa5f..a7307a9 100644 --- a/src/ShellCheck/Formatter/GCC.hs +++ b/src/ShellCheck/Formatter/GCC.hs @@ -22,6 +22,7 @@ module ShellCheck.Formatter.GCC (format) where import ShellCheck.Interface import ShellCheck.Formatter.Format +import Data.Either import Data.List import GHC.Exts import System.IO @@ -44,7 +45,7 @@ outputAll cr sys = mapM_ f groups f group = do let filename = sourceFile (head group) result <- (siReadFile sys) filename - let contents = either (const "") id result + let contents = fromRight "" result outputResult filename contents group outputResult filename contents warnings = do diff --git a/src/ShellCheck/Formatter/JSON1.hs b/src/ShellCheck/Formatter/JSON1.hs index 7335d8c..edc1974 100644 --- a/src/ShellCheck/Formatter/JSON1.hs +++ b/src/ShellCheck/Formatter/JSON1.hs @@ -24,6 +24,7 @@ import ShellCheck.Interface import ShellCheck.Formatter.Format import Data.Aeson +import Data.Either import Data.IORef import Data.Monoid import GHC.Exts @@ -118,7 +119,7 @@ collectResult ref cr sys = mapM_ f groups f group = do let filename = sourceFile (head group) result <- siReadFile sys filename - let contents = either (const "") id result + let contents = fromRight "" result let comments' = makeNonVirtual comments contents modifyIORef ref (\x -> comments' ++ x) diff --git a/src/ShellCheck/Formatter/TTY.hs b/src/ShellCheck/Formatter/TTY.hs index 4dabf45..ddf36aa 100644 --- a/src/ShellCheck/Formatter/TTY.hs +++ b/src/ShellCheck/Formatter/TTY.hs @@ -25,6 +25,7 @@ import ShellCheck.Formatter.Format import Control.Monad import Data.Array +import Data.Either import Data.Foldable import Data.Ord import Data.IORef @@ -122,7 +123,7 @@ outputResult options ref result sys = do outputForFile color sys comments = do let fileName = sourceFile (head comments) result <- (siReadFile sys) fileName - let contents = either (const "") id result + let contents = fromRight "" result let fileLinesList = lines contents let lineCount = length fileLinesList let fileLines = listArray (1, lineCount) fileLinesList From 1c6202dba4d31feb1a4e483df849a74f39e790e3 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 22:25:19 -0400 Subject: [PATCH 043/459] Avoid some awkward parentheses with forM_ --- src/ShellCheck/Formatter/TTY.hs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/ShellCheck/Formatter/TTY.hs b/src/ShellCheck/Formatter/TTY.hs index ddf36aa..44bda1e 100644 --- a/src/ShellCheck/Formatter/TTY.hs +++ b/src/ShellCheck/Formatter/TTY.hs @@ -128,7 +128,7 @@ outputForFile color sys comments = do let lineCount = length fileLinesList let fileLines = listArray (1, lineCount) fileLinesList let groups = groupWith lineNo comments - mapM_ (\commentsForLine -> do + forM_ groups $ \commentsForLine -> do let lineNum = fromIntegral $ lineNo (head commentsForLine) let line = if lineNum < 1 || lineNum > lineCount then "" @@ -137,10 +137,9 @@ outputForFile color sys comments = do putStrLn $ color "message" $ "In " ++ fileName ++" line " ++ show lineNum ++ ":" putStrLn (color "source" line) - mapM_ (\c -> putStrLn (color (severityText c) $ cuteIndent c)) commentsForLine + forM_ commentsForLine $ \c -> putStrLn $ color (severityText c) $ cuteIndent c putStrLn "" showFixedString color commentsForLine (fromIntegral lineNum) fileLines - ) groups -- Pick out only the lines necessary to show a fix in action sliceFile :: Fix -> Array Int String -> (Fix, Array Int String) From 3e17a2096595f004534ae29920be60f239e31fab Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sat, 11 Apr 2020 17:29:28 -0400 Subject: [PATCH 044/459] Simplify thenSkip, and use in another location --- src/ShellCheck/Parser.hs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 64d75c6..b96aee7 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -373,10 +373,7 @@ parseNoteAtId id c l a = do parseNoteAtWithEnd start end c l a = addParseNote $ ParseNote start end c l a --------- Convenient combinators -thenSkip main follow = do - r <- main - optional follow - return r +thenSkip main follow = main <* optional follow unexpecting s p = try $ (try p >> fail ("Unexpected " ++ s)) <|> return () @@ -420,7 +417,7 @@ acceptButWarn parser level code note = parsecBracket before after op = do val <- before - (op val <* optional (after val)) <|> (after val *> fail "") + op val `thenSkip` after val <|> (after val *> fail "") swapContext contexts p = parsecBracket (getCurrentContexts <* setCurrentContexts contexts) From 8a7497c4f02304811b8ce5ec15d3263fc78ba83e Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 17:07:05 -0400 Subject: [PATCH 045/459] Simplify checkVariableBraces --- src/ShellCheck/Analytics.hs | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index f5a8047..dd4e4c7 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1955,19 +1955,16 @@ prop_CheckVariableBraces1 = verify checkVariableBraces "a='123'; echo $a" prop_CheckVariableBraces2 = verifyNot checkVariableBraces "a='123'; echo ${a}" prop_CheckVariableBraces3 = verifyNot checkVariableBraces "#shellcheck disable=SC2016\necho '$a'" prop_CheckVariableBraces4 = verifyNot checkVariableBraces "echo $* $1" -checkVariableBraces params t = - case t of - T_DollarBraced id False _ - | name `notElem` unbracedVariables -> - styleWithFix id 2250 - "Prefer putting braces around variable references even when not strictly required." - (fixFor t) - - _ -> return () +checkVariableBraces params t@(T_DollarBraced id False _) + | name `notElem` unbracedVariables = + styleWithFix id 2250 + "Prefer putting braces around variable references even when not strictly required." + (fixFor t) where name = getBracedReference $ bracedString t fixFor token = fixWith [replaceStart (getId token) params 1 "${" ,replaceEnd (getId token) params 0 "}"] +checkVariableBraces _ _ = return () prop_checkQuotesInLiterals1 = verifyTree checkQuotesInLiterals "param='--foo=\"bar\"'; app $param" prop_checkQuotesInLiterals1a= verifyTree checkQuotesInLiterals "param=\"--foo='lolbar'\"; app $param" From a9d564a8bc8bbdd624543b2e06082e9bdc838a1b Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 17:19:14 -0400 Subject: [PATCH 046/459] Combine bracedString into getSingleUnmodifiedVariable Everywhere we used getSingleUnmodifiedVariable, we just called bracedString on the result. Move this into that function instead, and rename it accordingly. --- src/ShellCheck/Checks/Commands.hs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index e321102..4e9c045 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -687,8 +687,7 @@ prop_checkExportedExpansions4 = verifyNot checkExportedExpansions "export ${foo? checkExportedExpansions = CommandCheck (Exactly "export") (mapM_ check . arguments) where check t = sequence_ $ do - var <- getSingleUnmodifiedVariable t - let name = bracedString var + name <- getSingleUnmodifiedBracedString t return . warn (getId t) 2163 $ "This does not export '" ++ name ++ "'. Remove $/${} for that, or use ${var?} to quiet." @@ -709,21 +708,20 @@ checkReadExpansions = CommandCheck (Exactly "read") check check cmd = mapM_ warning $ getVars cmd warning t = sequence_ $ do - var <- getSingleUnmodifiedVariable t - let name = bracedString var + name <- getSingleUnmodifiedBracedString t guard $ isVariableName name -- e.g. not $1 return . warn (getId t) 2229 $ "This does not read '" ++ name ++ "'. Remove $/${} for that, or use ${var?} to quiet." -- Return the single variable expansion that makes up this word, if any. -- e.g. $foo -> $foo, "$foo"'' -> $foo , "hello $name" -> Nothing -getSingleUnmodifiedVariable :: Token -> Maybe Token -getSingleUnmodifiedVariable word = +getSingleUnmodifiedBracedString :: Token -> Maybe String +getSingleUnmodifiedBracedString word = case getWordParts word of [t@(T_DollarBraced {})] -> let contents = bracedString t name = getBracedReference contents - in guard (contents == name) >> return t + in guard (contents == name) >> return (bracedString t) _ -> Nothing prop_checkAliasesUsesArgs1 = verify checkAliasesUsesArgs "alias a='cp $1 /a'" From 999b7e259683ed63c2102e82dfe9d548f1564cbc Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 17:20:49 -0400 Subject: [PATCH 047/459] Get rid of bracedString everywhere it's easy to --- src/ShellCheck/ASTLib.hs | 8 ++++---- src/ShellCheck/Analytics.hs | 24 ++++++++++++------------ src/ShellCheck/AnalyzerLib.hs | 8 ++++---- src/ShellCheck/Checks/Commands.hs | 6 +++--- src/ShellCheck/Checks/ShellSupport.hs | 8 ++++---- 5 files changed, 27 insertions(+), 27 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 2b40705..4f49243 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -139,8 +139,8 @@ bracedString (T_DollarBraced _ _ l) = concat $ oversimplify l bracedString _ = error "Internal shellcheck error, please report! (bracedString on non-variable)" -- Is this an expansion of multiple items of an array? -isArrayExpansion t@(T_DollarBraced _ _ _) = - let string = bracedString t in +isArrayExpansion (T_DollarBraced _ _ l) = + let string = concat $ oversimplify l in "@" `isPrefixOf` string || not ("#" `isPrefixOf` string) && "[@]" `isInfixOf` string isArrayExpansion _ = False @@ -148,8 +148,8 @@ isArrayExpansion _ = False -- Is it possible that this arg becomes multiple args? mayBecomeMultipleArgs t = willBecomeMultipleArgs t || f t where - f t@(T_DollarBraced _ _ _) = - let string = bracedString t in + f (T_DollarBraced _ _ l) = + let string = concat $ oversimplify l in "!" `isPrefixOf` string f (T_DoubleQuoted _ parts) = any f parts f (T_NormalWord _ parts) = any f parts diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index dd4e4c7..23852b1 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -781,8 +781,8 @@ checkShorthandIf _ _ = return () prop_checkDollarStar = verify checkDollarStar "for f in $*; do ..; done" prop_checkDollarStar2 = verifyNot checkDollarStar "a=$*" prop_checkDollarStar3 = verifyNot checkDollarStar "[[ $* = 'a b' ]]" -checkDollarStar p t@(T_NormalWord _ [b@(T_DollarBraced id _ _)]) - | bracedString b == "*" && +checkDollarStar p t@(T_NormalWord _ [T_DollarBraced id _ l]) + | concat (oversimplify l) == "*" && not (isStrictlyQuoteFree (parentMap p) t) = warn id 2048 "Use \"$@\" (with quotes) to prevent whitespace problems." checkDollarStar _ _ = return () @@ -1309,8 +1309,8 @@ prop_checkArithmeticDeref13= verifyNot checkArithmeticDeref "(( $$ ))" prop_checkArithmeticDeref14= verifyNot checkArithmeticDeref "(( $! ))" prop_checkArithmeticDeref15= verifyNot checkArithmeticDeref "(( ${!var} ))" prop_checkArithmeticDeref16= verifyNot checkArithmeticDeref "(( ${x+1} + ${x=42} ))" -checkArithmeticDeref params t@(TA_Expansion _ [b@(T_DollarBraced id _ _)]) = - unless (isException $ bracedString b) getWarning +checkArithmeticDeref params t@(TA_Expansion _ [T_DollarBraced id _ l]) = + unless (isException $ concat $ oversimplify l) getWarning where isException [] = True isException s@(h:_) = any (`elem` "/.:#%?*@$-!+=^,") s || isDigit h @@ -1940,7 +1940,7 @@ checkSpacefulness' onFind params t = T_DollarArithmetic _ _ -> SpaceNone T_Literal _ s -> fromLiteral s T_SingleQuoted _ s -> fromLiteral s - T_DollarBraced _ _ _ -> spacefulF $ getBracedReference $ bracedString x + T_DollarBraced _ _ l -> spacefulF $ getBracedReference $ concat $ oversimplify l T_NormalWord _ w -> isSpacefulWord spacefulF w T_DoubleQuoted _ w -> isSpacefulWord spacefulF w _ -> SpaceEmpty @@ -1955,13 +1955,13 @@ prop_CheckVariableBraces1 = verify checkVariableBraces "a='123'; echo $a" prop_CheckVariableBraces2 = verifyNot checkVariableBraces "a='123'; echo ${a}" prop_CheckVariableBraces3 = verifyNot checkVariableBraces "#shellcheck disable=SC2016\necho '$a'" prop_CheckVariableBraces4 = verifyNot checkVariableBraces "echo $* $1" -checkVariableBraces params t@(T_DollarBraced id False _) +checkVariableBraces params t@(T_DollarBraced id False l) | name `notElem` unbracedVariables = styleWithFix id 2250 "Prefer putting braces around variable references even when not strictly required." (fixFor t) where - name = getBracedReference $ bracedString t + name = getBracedReference $ concat $ oversimplify l fixFor token = fixWith [replaceStart (getId token) params 1 "${" ,replaceEnd (getId token) params 0 "}"] checkVariableBraces _ _ = return () @@ -2010,7 +2010,7 @@ checkQuotesInLiterals params t = squashesQuotes t = case t of - T_DollarBraced id _ _ -> "#" `isPrefixOf` bracedString t + T_DollarBraced id _ l -> "#" `isPrefixOf` concat (oversimplify l) _ -> False readF _ expr name = do @@ -2271,7 +2271,7 @@ checkUnassignedReferences' includeGlobals params t = warnings isInArray var t = any isArray $ getPath (parentMap params) t where isArray T_Array {} = True - isArray b@(T_DollarBraced _ _ _) | var /= getBracedReference (bracedString b) = True + isArray (T_DollarBraced _ _ l) | var /= getBracedReference (concat $ oversimplify l) = True isArray _ = False isGuarded (T_DollarBraced _ _ v) = @@ -2399,7 +2399,7 @@ prop_checkPrefixAssign2 = verifyNot checkPrefixAssignmentReference "var=$(echo $ checkPrefixAssignmentReference params t@(T_DollarBraced id _ value) = check path where - name = getBracedReference $ bracedString t + name = getBracedReference $ concat $ oversimplify value path = getPath (parentMap params) t idPath = map getId path @@ -3032,7 +3032,7 @@ checkReturnAgainstZero _ token = isZero t = getLiteralString t == Just "0" isExitCode t = case getWordParts t of - [exp@T_DollarBraced {}] -> bracedString exp == "?" + [T_DollarBraced _ _ l] -> concat (oversimplify l) == "?" _ -> False message id = style id 2181 "Check exit code directly with e.g. 'if mycmd;', not indirectly with $?." @@ -3223,7 +3223,7 @@ checkSplittingInArrays params t = T_DollarBraced id _ str | not (isCountingReference part) && not (isQuotedAlternativeReference part) - && getBracedReference (bracedString part) `notElem` variablesWithoutSpaces + && getBracedReference (concat $ oversimplify str) `notElem` variablesWithoutSpaces -> warn id 2206 $ if shellType params == Ksh then "Quote to prevent word splitting/globbing, or split robustly with read -A or while read." diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 3a7f7da..69ebfe6 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -498,7 +498,7 @@ getModifiedVariables t = return (t, token, str, DataString SourceChecked) T_DollarBraced _ _ l -> do - let string = bracedString t + let string = concat $ oversimplify l let modifier = getBracedModifier string guard $ any (`isPrefixOf` modifier) ["=", ":="] return (t, t, getBracedReference string, DataString $ SourceFrom [l]) @@ -703,7 +703,7 @@ getOffsetReferences mods = fromMaybe [] $ do getReferencedVariables parents t = case t of - T_DollarBraced id _ l -> let str = bracedString t in + T_DollarBraced id _ l -> let str = concat $ oversimplify l in (t, t, getBracedReference str) : map (\x -> (l, l, x)) ( getIndexReferences str @@ -895,8 +895,8 @@ isCountingReference _ = False -- FIXME: doesn't handle ${a:+$var} vs ${a:+"$var"} isQuotedAlternativeReference t = case t of - T_DollarBraced _ _ _ -> - getBracedModifier (bracedString t) `matches` re + T_DollarBraced _ _ l -> + getBracedModifier (concat $ oversimplify l) `matches` re _ -> False where re = mkRegex "(^|\\]):?\\+" diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 4e9c045..3ac14c3 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -718,10 +718,10 @@ checkReadExpansions = CommandCheck (Exactly "read") check getSingleUnmodifiedBracedString :: Token -> Maybe String getSingleUnmodifiedBracedString word = case getWordParts word of - [t@(T_DollarBraced {})] -> - let contents = bracedString t + [T_DollarBraced _ _ l] -> + let contents = concat $ oversimplify l name = getBracedReference contents - in guard (contents == name) >> return (bracedString t) + in guard (contents == name) >> return contents _ -> Nothing prop_checkAliasesUsesArgs1 = verify checkAliasesUsesArgs "alias a='cp $1 /a'" diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 49d3212..d9deff6 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -243,7 +243,7 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do when (isBashVariable var) $ warnMsg id $ var ++ " is" where - str = bracedString t + str = concat $ oversimplify token var = getBracedReference str check (regex, feature) = when (isJust $ matchRegex regex str) $ warnMsg id feature @@ -506,13 +506,13 @@ checkMultiDimensionalArrays = ForShell [Bash] f case token of T_Assignment _ _ name (first:second:_) _ -> about second T_IndexedElement _ (first:second:_) _ -> about second - T_DollarBraced {} -> - when (isMultiDim token) $ about token + T_DollarBraced _ _ l -> + when (isMultiDim l) $ about token _ -> return () about t = warn (getId t) 2180 "Bash does not support multidimensional arrays. Use 1D or associative arrays." re = mkRegex "^\\[.*\\]\\[.*\\]" -- Fixme, this matches ${foo:- [][]} and such as well - isMultiDim t = getBracedModifier (bracedString t) `matches` re + isMultiDim l = getBracedModifier (concat $ oversimplify l) `matches` re prop_checkPS11 = verify checkPS1Assignments "PS1='\\033[1;35m\\$ '" prop_checkPS11a= verify checkPS1Assignments "export PS1='\\033[1;35m\\$ '" From b58bb4ba9d338d1b2ce8cf690deeb6ac43ce97a1 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 5 Apr 2020 17:26:24 -0400 Subject: [PATCH 048/459] Move bracedString to be local to its last use site --- src/ShellCheck/ASTLib.hs | 4 ---- src/ShellCheck/Analytics.hs | 4 ++++ 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 4f49243..efa8cdb 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -134,10 +134,6 @@ isUnquotedFlag token = fromMaybe False $ do str <- getLeadingUnquotedString token return $ "-" `isPrefixOf` str --- Given a T_DollarBraced, return a simplified version of the string contents. -bracedString (T_DollarBraced _ _ l) = concat $ oversimplify l -bracedString _ = error "Internal shellcheck error, please report! (bracedString on non-variable)" - -- Is this an expansion of multiple items of an array? isArrayExpansion (T_DollarBraced _ _ l) = let string = concat $ oversimplify l in diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 23852b1..dd98370 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1869,6 +1869,10 @@ checkSpacefulness params = checkSpacefulness' onFind params any (`isPrefixOf` modifier) ["=", ":="] && isParamTo parents ":" token + -- Given a T_DollarBraced, return a simplified version of the string contents. + bracedString (T_DollarBraced _ _ l) = concat $ oversimplify l + bracedString _ = error "Internal shellcheck error, please report! (bracedString on non-variable)" + prop_checkSpacefulness4v= verifyTree checkVerboseSpacefulness "foo=3; foo=$(echo $foo)" prop_checkSpacefulness8v= verifyTree checkVerboseSpacefulness "a=foo\\ bar; a=foo; rm $a" prop_checkSpacefulness28v = verifyTree checkVerboseSpacefulness "exec {n}>&1; echo $n" From 163c710ba7da07334999bf18804c971a2be41368 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 12 Apr 2020 16:15:45 -0400 Subject: [PATCH 049/459] Clean up and optimize getSuspiciousRegexWildcard --- src/ShellCheck/Checks/Commands.hs | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index c68bdd1..395ed2b 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -283,17 +283,11 @@ checkGrepRe = CommandCheck (Basename "grep") check where candidates = sampleWords ++ map (\(x:r) -> toUpper x : r) sampleWords - getSuspiciousRegexWildcard str = - if not $ str `matches` contra - then do - match <- matchRegex suspicious str - str <- match !!! 0 - str !!! 0 - else - fail "looks good" - where - suspicious = mkRegex "([A-Za-z1-9])\\*" - contra = mkRegex "[^a-zA-Z1-9]\\*|[][^$+\\\\]" + getSuspiciousRegexWildcard str = case matchRegex suspicious str of + Just [[c]] | not (str `matches` contra) -> Just c + _ -> fail "looks good" + suspicious = mkRegex "([A-Za-z1-9])\\*" + contra = mkRegex "[^a-zA-Z1-9]\\*|[][^$+\\\\]" prop_checkTrapQuotes1 = verify checkTrapQuotes "trap \"echo $num\" INT" From e0daa936d2b82f7a5993047bb95f9eb3eaa4db01 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Fri, 24 Apr 2020 22:14:08 -0400 Subject: [PATCH 050/459] Revert "Use fromRight instead of reimplementing it" We still support GHC 8.0, which didn't have fromRight. This reverts commit 64c31d91422e3c48cc7ed0176b069fa11456cee8. --- src/ShellCheck/Formatter/GCC.hs | 3 +-- src/ShellCheck/Formatter/JSON1.hs | 3 +-- src/ShellCheck/Formatter/TTY.hs | 3 +-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/ShellCheck/Formatter/GCC.hs b/src/ShellCheck/Formatter/GCC.hs index a7307a9..9c5fa5f 100644 --- a/src/ShellCheck/Formatter/GCC.hs +++ b/src/ShellCheck/Formatter/GCC.hs @@ -22,7 +22,6 @@ module ShellCheck.Formatter.GCC (format) where import ShellCheck.Interface import ShellCheck.Formatter.Format -import Data.Either import Data.List import GHC.Exts import System.IO @@ -45,7 +44,7 @@ outputAll cr sys = mapM_ f groups f group = do let filename = sourceFile (head group) result <- (siReadFile sys) filename - let contents = fromRight "" result + let contents = either (const "") id result outputResult filename contents group outputResult filename contents warnings = do diff --git a/src/ShellCheck/Formatter/JSON1.hs b/src/ShellCheck/Formatter/JSON1.hs index edc1974..7335d8c 100644 --- a/src/ShellCheck/Formatter/JSON1.hs +++ b/src/ShellCheck/Formatter/JSON1.hs @@ -24,7 +24,6 @@ import ShellCheck.Interface import ShellCheck.Formatter.Format import Data.Aeson -import Data.Either import Data.IORef import Data.Monoid import GHC.Exts @@ -119,7 +118,7 @@ collectResult ref cr sys = mapM_ f groups f group = do let filename = sourceFile (head group) result <- siReadFile sys filename - let contents = fromRight "" result + let contents = either (const "") id result let comments' = makeNonVirtual comments contents modifyIORef ref (\x -> comments' ++ x) diff --git a/src/ShellCheck/Formatter/TTY.hs b/src/ShellCheck/Formatter/TTY.hs index 44bda1e..0d474d7 100644 --- a/src/ShellCheck/Formatter/TTY.hs +++ b/src/ShellCheck/Formatter/TTY.hs @@ -25,7 +25,6 @@ import ShellCheck.Formatter.Format import Control.Monad import Data.Array -import Data.Either import Data.Foldable import Data.Ord import Data.IORef @@ -123,7 +122,7 @@ outputResult options ref result sys = do outputForFile color sys comments = do let fileName = sourceFile (head comments) result <- (siReadFile sys) fileName - let contents = fromRight "" result + let contents = either (const "") id result let fileLinesList = lines contents let lineCount = length fileLinesList let fileLines = listArray (1, lineCount) fileLinesList From 60e80e4ce1323b4f5944018dd5e132b0287cffd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Skytt=C3=A4?= Date: Sat, 25 Apr 2020 08:29:38 +0300 Subject: [PATCH 051/459] Spelling fixes --- src/ShellCheck/Parser.hs | 2 +- test/check_release | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index b96aee7..46d7ace 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -2622,7 +2622,7 @@ readFunctionDefinition = called "function" $ do readWithoutFunction = try $ do name <- many1 functionChars - guard $ name /= "time" -- Interfers with time ( foo ) + guard $ name /= "time" -- Interferes with time ( foo ) spacing readParens return $ \id -> T_Function id (FunctionKeyword False) (FunctionParentheses True) name diff --git a/test/check_release b/test/check_release index b9cd34b..91001fb 100755 --- a/test/check_release +++ b/test/check_release @@ -9,7 +9,7 @@ fail() { if git diff | grep -q "" then - fail "There are uncommited changes" + fail "There are uncommitted changes" fi current=$(git tag --points-at) From 1d126960f342f8cdda7fcc5e083284c3196e2fa2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Skytt=C3=A4?= Date: Sat, 25 Apr 2020 08:33:10 +0300 Subject: [PATCH 052/459] Use SC prefix for disable= in man page --- shellcheck.1.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shellcheck.1.md b/shellcheck.1.md index 50eaddc..6d2a732 100644 --- a/shellcheck.1.md +++ b/shellcheck.1.md @@ -254,7 +254,7 @@ Valid keys are: **shell** : Overrides the shell detected from the shebang. This is useful for files meant to be included (and thus lacking a shebang), or possibly - as a more targeted alternative to 'disable=2039'. + as a more targeted alternative to 'disable=SC2039'. # RC FILES From 2030b83607a84a84a96bbc73d8c541fdaa950fa5 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 3 May 2020 11:29:00 -0700 Subject: [PATCH 053/459] Warn about duplicate uses of stdin/out/err --- CHANGELOG.md | 3 + src/ShellCheck/Analytics.hs | 160 +++++++++++++++++++++++++++++++----- 2 files changed, 142 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 181cc54..fbc2ab7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,7 @@ ## Git +### Added +- SC2259/SC2260: Warn when redirections override pipes +- SC2261: Warn about multiple competing redirections ## v0.7.1 - 2020-04-04 ### Fixed diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 125835a..4281ace 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -3282,25 +3282,92 @@ prop_checkPipeToNowhere6 = verifyNot checkPipeToNowhere "ls | echo $(cat)" prop_checkPipeToNowhere7 = verifyNot checkPipeToNowhere "echo foo | var=$(cat) ls" prop_checkPipeToNowhere8 = verify checkPipeToNowhere "foo | true" prop_checkPipeToNowhere9 = verifyNot checkPipeToNowhere "mv -i f . < /dev/stdin" +prop_checkPipeToNowhere10 = verify checkPipeToNowhere "ls > file | grep foo" +prop_checkPipeToNowhere11 = verify checkPipeToNowhere "ls | grep foo < file" +prop_checkPipeToNowhere12 = verify checkPipeToNowhere "ls > foo > bar" +prop_checkPipeToNowhere13 = verify checkPipeToNowhere "ls > foo 2> bar > baz" +prop_checkPipeToNowhere14 = verify checkPipeToNowhere "ls > foo &> bar" +prop_checkPipeToNowhere15 = verifyNot checkPipeToNowhere "ls > foo 2> bar |& grep 'No space left'" +prop_checkPipeToNowhere16 = verifyNot checkPipeToNowhere "echo World | cat << EOF\nhello $(cat)\nEOF\n" +prop_checkPipeToNowhere17 = verify checkPipeToNowhere "echo World | cat << 'EOF'\nhello $(cat)\nEOF\n" +prop_checkPipeToNowhere18 = verifyNot checkPipeToNowhere "ls 1>&3 3>&1 3>&- | wc -l" + +data PipeType = StdoutPipe | StdoutStderrPipe | NoPipe deriving (Eq) checkPipeToNowhere :: Parameters -> Token -> WriterT [TokenComment] Identity () -checkPipeToNowhere _ t = +checkPipeToNowhere params t = case t of - T_Pipeline _ _ (first:rest) -> mapM_ checkPipe rest + T_Pipeline _ pipes cmds -> + mapM_ checkPipe $ commandsWithContext pipes cmds T_Redirecting _ redirects cmd | any redirectsStdin redirects -> checkRedir cmd _ -> return () where - checkPipe redir = sequence_ $ do - cmd <- getCommand redir - name <- getCommandBasename cmd - guard $ name `elem` nonReadingCommands - guard . not $ hasAdditionalConsumers cmd - -- Confusing echo for cat is so common that it's worth a special case - let suggestion = - if name == "echo" - then "Did you want 'cat' instead?" - else "Wrong command or missing xargs?" - return $ warn (getId cmd) 2216 $ - "Piping to '" ++ name ++ "', a command that doesn't read stdin. " ++ suggestion + checkPipe (input, stage, output) = do + let hasConsumers = hasAdditionalConsumers stage + let hasProducers = hasAdditionalProducers stage + + sequence_ $ do + cmd <- getCommand stage + name <- getCommandBasename cmd + guard $ name `elem` nonReadingCommands + guard $ not hasConsumers && input /= NoPipe + + -- Confusing echo for cat is so common that it's worth a special case + let suggestion = + if name == "echo" + then "Did you want 'cat' instead?" + else "Wrong command or missing xargs?" + return $ warn (getId cmd) 2216 $ + "Piping to '" ++ name ++ "', a command that doesn't read stdin. " ++ suggestion + + sequence_ $ do + T_Redirecting _ redirs cmd <- return stage + fds <- sequence $ map getRedirectionFds redirs + + let fdAndToken :: [(Integer, Token)] + fdAndToken = + concatMap (\(list, redir) -> map (\n -> (n, redir)) list) $ + zip fds redirs + + let fdMap = + Map.fromListWith (++) $ + map (\(a,b) -> (a,[b])) fdAndToken + + let inputWarning = sequence_ $ do + guard $ input /= NoPipe && not hasConsumers + (override:_) <- Map.lookup 0 fdMap + return $ err (getOpId override) 2259 $ + "This redirection overrides piped input. To use both, merge or pass filename." + + -- Only produce output warnings for regular pipes, since these are + -- way more common, and `foo > out 2> err |& foo` can still write + -- to stderr if the files fail to open + let outputWarning = sequence_ $ do + guard $ output == StdoutPipe && not hasProducers + (override:_) <- Map.lookup 1 fdMap + return $ err (getOpId override) 2260 $ + "This redirection overrides the output pipe. Use 'tee' to output to both." + + return $ do + inputWarning + outputWarning + mapM_ warnAboutDupes $ Map.assocs fdMap + + warnAboutDupes (n, list@(_:_:_)) = + forM_ list $ \c -> err (getOpId c) 2261 $ + "Multiple redirections compete for " ++ str n ++ ". Combine, or use " ++ alternative ++ "." + warnAboutDupes _ = return () + + alternative = + if shellType params `elem` [Bash, Ksh] + then "process substitutions or temp files" + else "temporary files" + + str n = + case n of + 0 -> "stdin" + 1 -> "stdout" + 2 -> "stderr" + _ -> "FD " ++ show n checkRedir cmd = sequence_ $ do name <- getCommandBasename cmd @@ -3315,23 +3382,74 @@ checkPipeToNowhere _ t = "Redirecting to '" ++ name ++ "', a command that doesn't read stdin. " ++ suggestion -- Could any words in a SimpleCommand consume stdin (e.g. echo "$(cat)")? - hasAdditionalConsumers t = isNothing $ - doAnalysis (guard . not . mayConsume) t + hasAdditionalConsumers = treeContains mayConsume + -- Could any words in a SimpleCommand produce stdout? E.g. >(tee foo) + hasAdditionalProducers = treeContains mayProduce + treeContains pred t = isNothing $ + doAnalysis (guard . not . pred) t mayConsume t = case t of - T_ProcSub {} -> True + T_ProcSub _ "<" _ -> True T_Backticked {} -> True T_DollarExpansion {} -> True _ -> False - redirectsStdin t = + mayProduce t = case t of - T_FdRedirect _ _ (T_IoFile _ T_Less {} _) -> True - T_FdRedirect _ _ T_HereDoc {} -> True - T_FdRedirect _ _ T_HereString {} -> True + T_ProcSub _ ">" _ -> True _ -> False + getOpId t = + case t of + T_FdRedirect _ _ x -> getOpId x + T_IoFile _ op _ -> getId op + _ -> getId t + + getRedirectionFds t = + case t of + T_FdRedirect _ "" x -> getDefaultFds x + T_FdRedirect _ "&" _ -> return [1, 2] + T_FdRedirect _ num x | all isDigit num -> + -- Don't report the number unless we know what it is. + -- This avoids triggering on 3>&1 1>&3 + getDefaultFds x *> return [read num] + -- Don't bother with {fd}>42 and such + _ -> Nothing + + getDefaultFds redir = + case redir of + T_HereDoc {} -> return [0] + T_HereString {} -> return [0] + T_IoFile _ op _ -> + case op of + T_Less {} -> return [0] + T_Greater {} -> return [1] + T_DGREAT {} -> return [1] + T_GREATAND {} -> return [1, 2] + T_CLOBBER {} -> return [1] + T_IoDuplicate _ op "-" -> getDefaultFds op + _ -> Nothing + _ -> Nothing + + redirectsStdin t = + fromMaybe False $ do + fds <- getRedirectionFds t + return $ 0 `elem` fds + + pipeType t = + case t of + T_Pipe _ "|" -> StdoutPipe + T_Pipe _ "|&" -> StdoutStderrPipe + _ -> NoPipe + + commandsWithContext pipes cmds = + let pipeTypes = map pipeType pipes + inputs = NoPipe : pipeTypes + outputs = pipeTypes ++ [NoPipe] + in + zip3 inputs cmds outputs + prop_checkUseBeforeDefinition1 = verifyTree checkUseBeforeDefinition "f; f() { true; }" prop_checkUseBeforeDefinition2 = verifyNotTree checkUseBeforeDefinition "f() { true; }; f" prop_checkUseBeforeDefinition3 = verifyNotTree checkUseBeforeDefinition "if ! mycmd --version; then mycmd() { true; }; fi" From d6adbfde787a46e2d61dc96ffdf8c9a6f800390e Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 3 May 2020 21:46:16 -0700 Subject: [PATCH 054/459] Improve SC2259/60/61 messages --- src/ShellCheck/Analytics.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 4281ace..9c9a86d 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -3336,7 +3336,7 @@ checkPipeToNowhere params t = guard $ input /= NoPipe && not hasConsumers (override:_) <- Map.lookup 0 fdMap return $ err (getOpId override) 2259 $ - "This redirection overrides piped input. To use both, merge or pass filename." + "This redirection overrides piped input. To use both, merge or pass filenames." -- Only produce output warnings for regular pipes, since these are -- way more common, and `foo > out 2> err |& foo` can still write @@ -3354,7 +3354,7 @@ checkPipeToNowhere params t = warnAboutDupes (n, list@(_:_:_)) = forM_ list $ \c -> err (getOpId c) 2261 $ - "Multiple redirections compete for " ++ str n ++ ". Combine, or use " ++ alternative ++ "." + "Multiple redirections compete for " ++ str n ++ ". Use cat, tee, or pass filenames instead." warnAboutDupes _ = return () alternative = From c2a15ce8e906ae6a12bcbe32eac7ac586fbcb59b Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 3 May 2020 21:52:25 -0700 Subject: [PATCH 055/459] Allow disabling SC1072/SC1073 with annotations (fixes #1931) --- CHANGELOG.md | 5 +++++ src/ShellCheck/Checker.hs | 7 +++++++ src/ShellCheck/Parser.hs | 19 +++++++++++++++---- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fbc2ab7..6089c76 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,11 @@ - SC2259/SC2260: Warn when redirections override pipes - SC2261: Warn about multiple competing redirections +### Fixed +- SC1072/SC1073 now respond to disable annotations, though ignoring parse errors + is still purely cosmetic and does not allow ShellCheck to continue. + + ## v0.7.1 - 2020-04-04 ### Fixed - `-f diff` no longer claims that it found more issues when it didn't diff --git a/src/ShellCheck/Checker.hs b/src/ShellCheck/Checker.hs index 120f7d8..f639ab3 100644 --- a/src/ShellCheck/Checker.hs +++ b/src/ShellCheck/Checker.hs @@ -295,6 +295,13 @@ prop_canDisableShebangWarning = null $ result csScript = "#shellcheck disable=SC2148\nfoo" } +prop_canDisableParseErrors = null $ result + where + result = checkWithSpec [] emptyCheckSpec { + csFilename = "file.sh", + csScript = "#shellcheck disable=SC1073,SC1072,SC2148\n()" + } + prop_shExtensionDoesntMatter = result == [2148] where result = checkWithSpec [] emptyCheckSpec { diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 46d7ace..7eb2c32 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -253,7 +253,11 @@ ignoreProblemsOf p = do shouldIgnoreCode code = do context <- getCurrentContexts checkSourced <- Mr.asks checkSourced - return $ any (disabling checkSourced) context + return $ any (contextItemDisablesCode checkSourced code) context + +-- Does this item on the context stack disable warnings for 'code'? +contextItemDisablesCode :: Bool -> Integer -> Context -> Bool +contextItemDisablesCode alsoCheckSourced code = disabling alsoCheckSourced where disabling checkSourced item = case item of @@ -263,6 +267,8 @@ shouldIgnoreCode code = do disabling' (DisableComment n) = code == n disabling' _ = False + + getCurrentAnnotations includeSource = concatMap get . takeWhile (not . isBoundary) <$> getCurrentContexts where @@ -3313,16 +3319,21 @@ parseShell env name contents = do prRoot = Just $ reattachHereDocs script (hereDocMap userstate) } - Left err -> + Left err -> do + let context = contextStack state return newParseResult { prComments = map toPositionedComment $ - notesForContext (contextStack state) - ++ [makeErrorFor err] + (filter (not . isIgnored context) $ + notesForContext context + ++ [makeErrorFor err]) ++ parseProblems state, prTokenPositions = Map.empty, prRoot = Nothing } + where + -- A final pass for ignoring parse errors after failed parsing + isIgnored stack note = any (contextItemDisablesCode False (codeForParseNote note)) stack notesForContext list = zipWith ($) [first, second] $ filter isName list where From 536cb584f486fbd93feb2c3fe8f30703e52a6118 Mon Sep 17 00:00:00 2001 From: geeseven <2334728+geeseven@users.noreply.github.com> Date: Tue, 12 May 2020 14:58:15 -0500 Subject: [PATCH 056/459] update dependency free AUR package The `shellcheck-static` AUR package has been removed and `shellcheck-bin` is an option for a dependency free AUR package. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ce57d07..bfb3a99 100644 --- a/README.md +++ b/README.md @@ -148,7 +148,7 @@ On Arch Linux based distros: pacman -S shellcheck -or get the dependency free [shellcheck-static](https://aur.archlinux.org/packages/shellcheck-static/) from the AUR. +or get the dependency free [shellcheck-bin](https://aur.archlinux.org/packages/shellcheck-bin/) from the AUR. On Gentoo based distros: From a08ad3bee91179009a2daa34427148b03c885af6 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Fri, 8 May 2020 10:19:33 -0700 Subject: [PATCH 057/459] Count $# as an argument reference in SC2120 --- src/ShellCheck/Analytics.hs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 9c9a86d..fcc942b 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2555,6 +2555,7 @@ prop_checkUnpassedInFunctions10= verifyNotTree checkUnpassedInFunctions "foo() { prop_checkUnpassedInFunctions11= verifyNotTree checkUnpassedInFunctions "foo() { bar() { echo $1; }; bar baz; }; foo;" prop_checkUnpassedInFunctions12= verifyNotTree checkUnpassedInFunctions "foo() { echo ${!var*}; }; foo;" prop_checkUnpassedInFunctions13= verifyNotTree checkUnpassedInFunctions "# shellcheck disable=SC2120\nfoo() { echo $1; }\nfoo\n" +prop_checkUnpassedInFunctions14= verifyTree checkUnpassedInFunctions "foo() { echo $#; }; foo" checkUnpassedInFunctions params root = execWriter $ mapM_ warnForGroup referenceGroups where @@ -2596,7 +2597,7 @@ checkUnpassedInFunctions params root = return $ tell [(str, null args, t)] checkCommand _ = Nothing - isPositional str = str == "*" || str == "@" + isPositional str = str == "*" || str == "@" || str == "#" || (all isDigit str && str /= "0" && str /= "") isArgumentless (_, b, _) = b From 5cf2c00ff7d4d823b5cbe6420b2a48f1ed0447b3 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 25 May 2020 23:21:47 -0700 Subject: [PATCH 058/459] Warn about defining and using an alias in a single command (fixes #1807) --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 72 +++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6089c76..deacf5b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ### Added - SC2259/SC2260: Warn when redirections override pipes - SC2261: Warn about multiple competing redirections +- SC2262/SC2263: Warn about aliases declared and used in the same parsing unit ### Fixed - SC1072/SC1073 now respond to disable annotations, though ignoring parse errors diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index fcc942b..977d415 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -64,6 +64,7 @@ treeChecks = [ ,checkUncheckedCdPushdPopd ,checkArrayAssignmentIndices ,checkUseBeforeDefinition + ,checkAliasUsedInSameParsingUnit ] runAnalytics :: AnalysisSpec -> [TokenComment] @@ -3740,6 +3741,77 @@ checkModifiedArithmeticInRedirection params t = unless (shellType params == Dash warnFor id = warn id 2257 "Arithmetic modifications in command redirections may be discarded. Do them separately." +prop_checkAliasUsedInSameParsingUnit1 = verifyTree checkAliasUsedInSameParsingUnit "alias x=y; x" +prop_checkAliasUsedInSameParsingUnit2 = verifyNotTree checkAliasUsedInSameParsingUnit "alias x=y\nx" +prop_checkAliasUsedInSameParsingUnit3 = verifyTree checkAliasUsedInSameParsingUnit "{ alias x=y\nx\n}" +prop_checkAliasUsedInSameParsingUnit4 = verifyNotTree checkAliasUsedInSameParsingUnit "alias x=y; 'x';" +prop_checkAliasUsedInSameParsingUnit5 = verifyNotTree checkAliasUsedInSameParsingUnit ":\n{\n#shellcheck disable=SC2262\nalias x=y\nx\n}" +prop_checkAliasUsedInSameParsingUnit6 = verifyNotTree checkAliasUsedInSameParsingUnit ":\n{\n#shellcheck disable=SC2262\nalias x=y\nalias x=z\nx\n}" -- Only consider the first instance +checkAliasUsedInSameParsingUnit :: Parameters -> Token -> [TokenComment] +checkAliasUsedInSameParsingUnit params root = + let + -- Get all root commands + commands = concat $ getCommandSequences root + -- Group them by whether they start on the same line where the previous one ended + units = groupByLink followsOnLine commands + in + execWriter $ sequence_ $ map checkUnit units + where + lineSpan t = + let m = tokenPositions params in do + (start, end) <- Map.lookup t m + return $ (posLine start, posLine end) + + followsOnLine a b = fromMaybe False $ do + (_, end) <- lineSpan (getId a) + (start, _) <- lineSpan (getId b) + return $ end == start + + checkUnit :: [Token] -> Writer [TokenComment] () + checkUnit unit = evalStateT (mapM_ (doAnalysis findCommands) unit) (Map.empty) + + isSourced t = + let + f (T_SourceCommand {}) = True + f _ = False + in + any f $ getPath (parentMap params) t + + findCommands :: Token -> StateT (Map.Map String Token) (Writer [TokenComment]) () + findCommands t = case t of + T_SimpleCommand _ _ (cmd:args) -> + case getUnquotedLiteral cmd of + Just "alias" -> + mapM_ addAlias args + Just name | '/' `notElem` name -> do + cmd <- gets (Map.lookup name) + case cmd of + Just alias -> + unless (isSourced t || shouldIgnoreCode params 2262 alias) $ do + warn (getId alias) 2262 "This alias can't be defined and used in the same parsing unit. Use a function instead." + info (getId t) 2263 "Since they're in the same parsing unit, this command will not refer to the previously mentioned alias." + _ -> return () + _ -> return () + _ -> return () + addAlias arg = do + let (name, value) = break (== '=') $ getLiteralStringDef "-" arg + when (isVariableName name && not (null value)) $ + modify (Map.insertWith (\new old -> old) name arg) + +-- Like groupBy, but compares pairs of adjacent elements, rather than against the first of the span +prop_groupByLink1 = groupByLink (\a b -> a+1 == b) [1,2,3,2,3,7,8,9] == [[1,2,3], [2,3], [7,8,9]] +prop_groupByLink2 = groupByLink (==) ([] :: [()]) == [] +groupByLink :: (a -> a -> Bool) -> [a] -> [[a]] +groupByLink f list = + case list of + [] -> [] + (x:xs) -> g x [] xs + where + g current span (next:rest) = + if f current next + then g next (current:span) rest + else (reverse $ current:span) : g next [] rest + g current span [] = [reverse (current:span)] return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) From a2b5b6a50030d50da832e9f5c314d48e4746034f Mon Sep 17 00:00:00 2001 From: Onno Zweers Date: Wed, 3 Jun 2020 11:37:53 +0200 Subject: [PATCH 059/459] Rephrase: *Shellcheck* can't follow non-constant source The message "Can't follow non-constant source. Use a directive to specify location." set me off on the wrong foot. At first, I thought it meant "In bash, you can't source a script specified by a variable." It was only after reading the wiki page for this message https://github.com/koalaman/shellcheck/wiki/SC1090 that I understood that the problem is that *shellcheck* can't check the sourced file. So I would suggest to rephrase this message so that it is more clear that the problem is in the checking, not in the running of the script. --- src/ShellCheck/Parser.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 7eb2c32..01795ed 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -2129,7 +2129,7 @@ readSource t@(T_Redirecting _ _ (T_SimpleCommand cmdId _ (cmd:file':rest'))) = d case literalFile of Nothing -> do parseNoteAtId (getId file) WarningC 1090 - "Can't follow non-constant source. Use a directive to specify location." + "Shellcheck can't follow non-constant source. Use a directive to specify location." return t Just filename -> do proceed <- shouldFollow filename From 12d9c1b76d0b25637552e693b832fdcedf90238c Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 24 Jun 2020 11:50:27 -0700 Subject: [PATCH 060/459] Clarify that SC1090 refers to ShellCheck, not sh --- src/ShellCheck/Parser.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 7eb2c32..e8e04e0 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -2129,7 +2129,7 @@ readSource t@(T_Redirecting _ _ (T_SimpleCommand cmdId _ (cmd:file':rest'))) = d case literalFile of Nothing -> do parseNoteAtId (getId file) WarningC 1090 - "Can't follow non-constant source. Use a directive to specify location." + "ShellCheck can't follow non-constant source. Use a directive to specify location." return t Just filename -> do proceed <- shouldFollow filename From a61d8a232c403d73c071719292683e9e7cc24763 Mon Sep 17 00:00:00 2001 From: Aurelio Jargas Date: Fri, 26 Jun 2020 02:13:33 +0200 Subject: [PATCH 061/459] SC1102: Fix typo in error message: substition --- src/ShellCheck/Parser.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index e8e04e0..1f6b45b 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -1557,7 +1557,7 @@ readDollarExpression = do readDollarExp = arithmetic <|> readDollarExpansion <|> readDollarBracket <|> readDollarBraceCommandExpansion <|> readDollarBraced <|> readDollarVariable where arithmetic = readAmbiguous "$((" readDollarArithmetic readDollarExpansion (\pos -> - parseNoteAt pos ErrorC 1102 "Shells disambiguate $(( differently or not at all. For $(command substition), add space after $( . For $((arithmetics)), fix parsing errors.") + parseNoteAt pos ErrorC 1102 "Shells disambiguate $(( differently or not at all. For $(command substitution), add space after $( . For $((arithmetics)), fix parsing errors.") prop_readDollarSingleQuote = isOk readDollarSingleQuote "$'foo\\\'lol'" readDollarSingleQuote = called "$'..' expression" $ do From 6b88a341f33efc1957f6d0621b2acf80270a9fbd Mon Sep 17 00:00:00 2001 From: Stavros Ntentos <133706+stdedos@users.noreply.github.com> Date: Sun, 28 Jun 2020 01:47:02 +0300 Subject: [PATCH 062/459] Autolink https://www.shellcheck.net/ --- .github/ISSUE_TEMPLATE.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index b7f88e8..44d151e 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -2,10 +2,10 @@ - Rule Id (if any, e.g. SC1000): - My shellcheck version (`shellcheck --version` or "online"): - [ ] The rule's wiki page does not already cover this (e.g. https://shellcheck.net/wiki/SC2086) -- [ ] I tried on shellcheck.net and verified that this is still a problem on the latest commit +- [ ] I tried on https://www.shellcheck.net/ and verified that this is still a problem on the latest commit #### For new checks and feature suggestions -- [ ] shellcheck.net (i.e. the latest commit) currently gives no useful warnings about this +- [ ] https://www.shellcheck.net/ (i.e. the latest commit) currently gives no useful warnings about this - [ ] I searched through https://github.com/koalaman/shellcheck/issues and didn't find anything related From 739eaadbf515f4548b5d29e3611d70bd56ab7a4c Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 28 Jun 2020 16:01:15 -0700 Subject: [PATCH 063/459] Warn about extra spaces between ((s in for((;;)) --- src/ShellCheck/Parser.hs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index e8e04e0..b3b3425 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -2485,19 +2485,31 @@ readForClause = called "for loop" $ do readArithmetic id <|> readRegular id where readArithmetic id = called "arithmetic for condition" $ do - try $ string "((" + readArithmeticDelimiter '(' "Missing second '(' to start arithmetic for ((;;)) loop" x <- readArithmeticContents char ';' >> spacing y <- readArithmeticContents char ';' >> spacing z <- readArithmeticContents spacing - string "))" + readArithmeticDelimiter ')' "Missing second ')' to terminate 'for ((;;))' loop condition" spacing optional $ readSequentialSep >> spacing group <- readBraced <|> readDoGroup id return $ T_ForArithmetic id x y z group + -- For c='(' read "((" and be lenient about spaces + readArithmeticDelimiter c msg = do + char c + startPos <- getPosition + sp <- spacing + endPos <- getPosition + char c <|> do + parseProblemAt startPos ErrorC 1137 msg + fail "" + unless (null sp) $ + parseProblemAtWithEnd startPos endPos ErrorC 1138 $ "Remove spaces between " ++ [c,c] ++ " in arithmetic for loop." + readBraced = do (T_BraceGroup _ list) <- readBraceGroup return list From 210cdcd01a91923a7e1d04b4392616460ec5be68 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 28 Jun 2020 17:24:07 -0700 Subject: [PATCH 064/459] Treat $x/ or $(x)/ as ./ when finding sourced files (fixes #1998) --- CHANGELOG.md | 3 +++ src/ShellCheck/ASTLib.hs | 5 +++++ src/ShellCheck/Checker.hs | 6 ++++++ src/ShellCheck/Parser.hs | 12 +++++++++++- 4 files changed, 25 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index deacf5b..fde8b0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ - SC1072/SC1073 now respond to disable annotations, though ignoring parse errors is still purely cosmetic and does not allow ShellCheck to continue. +### Changed +- SC1090: A leading `$x/` or `$(x)/` is now treated as `./` when locating files + ## v0.7.1 - 2020-04-04 ### Fixed diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 4fdf078..29ce27f 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -503,6 +503,11 @@ isCommandSubstitution t = case t of T_Backticked {} -> True _ -> False +-- Is this an expansion that results in a simple string? +isStringExpansion t = isCommandSubstitution t || case t of + T_DollarArithmetic {} -> True + T_DollarBraced {} -> not (isArrayExpansion t) + _ -> False -- Is this a T_Annotation that ignores a specific code? isAnnotationIgnoringCode code t = diff --git a/src/ShellCheck/Checker.hs b/src/ShellCheck/Checker.hs index f639ab3..673a116 100644 --- a/src/ShellCheck/Checker.hs +++ b/src/ShellCheck/Checker.hs @@ -229,6 +229,12 @@ prop_cantSourceDynamic = prop_cantSourceDynamic2 = [1090] == checkWithIncludes [("lib", "")] "source ~/foo" +prop_canStripPrefixAndSource = + null $ checkWithIncludes [("./lib", "")] "source \"$MYDIR/lib\"" + +prop_canStripPrefixAndSource2 = + null $ checkWithIncludes [("./utils.sh", "")] "source \"$(dirname \"${BASH_SOURCE[0]}\")/utils.sh\"" + prop_canSourceDynamicWhenRedirected = null $ checkWithIncludes [("lib", "")] "#shellcheck source=lib\n. \"$1\"" diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 359fabd..bb27ea3 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -2122,7 +2122,7 @@ readSource t@(T_Redirecting _ _ (T_SimpleCommand cmdId _ (cmd:file':rest'))) = d let file = getFile file' rest' override <- getSourceOverride let literalFile = do - name <- override `mplus` getLiteralString file + name <- override `mplus` getLiteralString file `mplus` stripDynamicPrefix file -- Hack to avoid 'source ~/foo' trying to read from literal tilde guard . not $ "~/" `isPrefixOf` name return name @@ -2182,6 +2182,16 @@ readSource t@(T_Redirecting _ _ (T_SimpleCommand cmdId _ (cmd:file':rest'))) = d SourcePath x -> Just x _ -> Nothing + -- If the word has a single expansion as the directory, try stripping it + -- This affects `$foo/bar` but not `${foo}-dir/bar` or `/foo/$file` + stripDynamicPrefix word = + case getWordParts word of + exp : rest | isStringExpansion exp -> do + str <- getLiteralString (T_NormalWord (Id 0) rest) + guard $ "/" `isPrefixOf` str + return $ "." ++ str + _ -> Nothing + subRead name script = withContext (ContextSource name) $ inSeparateContext $ From baab5b53e09e82f1a4ead133f8ba904458e052ea Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 1 Jul 2020 21:21:38 -0700 Subject: [PATCH 065/459] Use TravisCI workspaces --- .prepare_deploy | 6 +++++- .travis.yml | 38 +++++++++++++++++++++++++++++++++++--- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/.prepare_deploy b/.prepare_deploy index 5e4ffaf..226c16d 100755 --- a/.prepare_deploy +++ b/.prepare_deploy @@ -1,7 +1,7 @@ #!/bin/bash # This script packages up Travis compiled binaries set -ex -shopt -s nullglob +shopt -s nullglob extglob cd deploy cp ../LICENSE LICENSE.txt @@ -59,7 +59,11 @@ do rm "shellcheck" done +rm !(*.xz|*.zip) + for file in ./* do sha512sum "$file" > "$file.sha512sum" done + +ls -l diff --git a/.travis.yml b/.travis.yml index 0ec9718..6e217e8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,14 +7,47 @@ services: jobs: include: - stage: Build - # This must weirdly not have a dash, otherwise an empty job is created env: BUILD=linux - - env: BUILD=windows + workspaces: + create: + name: ws-linux + paths: deploy +# - env: BUILD=windows +# workspaces: +# create: +# name: ws-windows +# paths: deploy - env: BUILD=armv6hf + workspaces: + create: + name: ws-armv6hf + paths: deploy - env: BUILD=aarch64 + workspaces: + create: + name: ws-aarch64 + paths: deploy - env: BUILD=osx os: osx + workspaces: + create: + name: ws-osx + paths: deploy + + - stage: Upload Artifacts to GitHub + workspaces: + use: + - ws-osx + - ws-linux + - ws-armv6hf + - ws-aarch64 + script: + - ls -la ${CASHER_DIR}/ || true + # Kludge broken TravisCI workspaces + - tar -xvf ${CASHER_DIR}/ws-osx-fetch.tgz --strip-components=5 + - ls -la deploy + - ./.github_deploy - stage: Deploy docker image # Deploy only for pushes to master branch, not other branches, not PRs. @@ -39,7 +72,6 @@ script: - ./striptests - set -ex; build_"$BUILD"; set +x; - ./.prepare_deploy - - ./.github_deploy # This is in global context and runs for every stage that doesn't override it. after_failure: | From 9793d94206eb1c8b28a00c2be6a47f6fa0ec7f49 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 5 Jul 2020 20:30:18 -0700 Subject: [PATCH 066/459] Remove trailing whitespace --- .compile_binaries | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.compile_binaries b/.compile_binaries index f657d76..95939ae 100755 --- a/.compile_binaries +++ b/.compile_binaries @@ -16,8 +16,8 @@ build_linux() { ls -l shellcheck ./shellcheck myscript for tag in $TAGS - do - cp "shellcheck" "deploy/shellcheck-$tag.linux-x86_64"; + do + cp "shellcheck" "deploy/shellcheck-$tag.linux-x86_64"; done } @@ -35,8 +35,8 @@ build_armv6hf() { # Linux armv6hf static executable docker run -v "$PWD:/mnt" koalaman/armv6hf-builder -c 'compile-shellcheck' for tag in $TAGS - do - cp "shellcheck" "deploy/shellcheck-$tag.linux-armv6hf"; + do + cp "shellcheck" "deploy/shellcheck-$tag.linux-armv6hf"; done } @@ -44,8 +44,8 @@ build_windows() { # Windows .exe docker run -v "$PWD:/appdata" koalaman/winghc cuib for tag in $TAGS - do - cp "dist/build/ShellCheck/shellcheck.exe" "deploy/shellcheck-$tag.exe"; + do + cp "dist/build/ShellCheck/shellcheck.exe" "deploy/shellcheck-$tag.exe"; done } @@ -66,7 +66,7 @@ build_osx() { [[ -e "$path" ]] for tag in $TAGS - do + do cp "$path" "deploy/shellcheck-$tag.darwin-x86_64"; done } From 7a9dbc042b5b41823fa09f80caedc5bc41deff03 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 5 Jul 2020 20:30:25 -0700 Subject: [PATCH 067/459] Re-enable Windows job --- .travis.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6e217e8..d7d202b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,11 +13,11 @@ jobs: create: name: ws-linux paths: deploy -# - env: BUILD=windows -# workspaces: -# create: -# name: ws-windows -# paths: deploy + - env: BUILD=windows + workspaces: + create: + name: ws-windows + paths: deploy - env: BUILD=armv6hf workspaces: create: @@ -42,6 +42,7 @@ jobs: - ws-linux - ws-armv6hf - ws-aarch64 + - ws-windows script: - ls -la ${CASHER_DIR}/ || true # Kludge broken TravisCI workspaces From 5b86777f9d0ee112b9555002ffe2d834510a25db Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 22 Jul 2020 17:32:00 -0700 Subject: [PATCH 068/459] Warn about non-POSIX case modification expansions (fixes #1977) --- src/ShellCheck/Checks/ShellSupport.hs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 6dd2d39..1f87868 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -178,6 +178,8 @@ prop_checkBashisms93 = verify checkBashisms "#!/bin/sh\necho $(( 10#$(date +%m) prop_checkBashisms94 = verify checkBashisms "#!/bin/sh\n[ -v var ]" prop_checkBashisms95 = verify checkBashisms "#!/bin/sh\necho $_" prop_checkBashisms96 = verifyNot checkBashisms "#!/bin/dash\necho $_" +prop_checkBashisms97 = verify checkBashisms "#!/bin/sh\necho ${var,}" +prop_checkBashisms98 = verify checkBashisms "#!/bin/sh\necho ${var^^}" checkBashisms = ForShell [Sh, Dash] $ \t -> do params <- ask kludge params t @@ -407,6 +409,7 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do (re $ "^![" ++ varChars ++ "]+[*@]$", "name matching prefixes are"), (re $ "^[" ++ varChars ++ "*@]+:[^-=?+]", "string indexing is"), (re $ "^([*@][%#]|#[@*])", "string operations on $@/$* are"), + (re $ "^[" ++ varChars ++ "*@]+(\\[.*\\])?[,^]", "case modification is"), (re $ "^[" ++ varChars ++ "*@]+(\\[.*\\])?/", "string replacement is") ] bashVars = [ From 5d753212fb8385ff80d10f8f8ad96a90e74cfd54 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 25 Jul 2020 13:45:05 -0700 Subject: [PATCH 069/459] Improve handling of command prefixes like exec/command (fixes #2008) --- src/ShellCheck/ASTLib.hs | 73 ++++++++++++++++++++++----- src/ShellCheck/Analytics.hs | 32 +++++++----- src/ShellCheck/AnalyzerLib.hs | 27 ---------- src/ShellCheck/Checks/ShellSupport.hs | 2 +- 4 files changed, 81 insertions(+), 53 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 29ce27f..09c7537 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -28,6 +28,7 @@ import Data.Functor import Data.Functor.Identity import Data.List import Data.Maybe +import qualified Data.Map as Map -- Is this a type of loop? isLoop t = case t of @@ -134,6 +135,33 @@ isUnquotedFlag token = fromMaybe False $ do str <- getLeadingUnquotedString token return $ "-" `isPrefixOf` str +-- getGnuOpts "erd:u:" will parse a SimpleCommand like +-- read -re -d : -u 3 bar +-- into +-- Just [("r", -re), ("e", -re), ("d", :), ("u", 3), ("", bar)] +-- where flags with arguments map to arguments, while others map to themselves. +-- Any unrecognized flag will result in Nothing. +getGnuOpts str t = getOpts str $ getAllFlags t +getBsdOpts str t = getOpts str $ getLeadingFlags t +getOpts :: String -> [(Token, String)] -> Maybe [(String, Token)] +getOpts string flags = process flags + where + flagList (c:':':rest) = ([c], True) : flagList rest + flagList (c:rest) = ([c], False) : flagList rest + flagList [] = [] + flagMap = Map.fromList $ ("", False) : flagList string + + process [] = return [] + process ((token1, flag):rest1) = do + takesArg <- Map.lookup flag flagMap + (token, rest) <- if takesArg + then case rest1 of + (token2, ""):rest2 -> return (token2, rest2) + _ -> fail "takesArg without valid arg" + else return (token1, rest1) + more <- process rest + return $ (flag, token) : more + -- Is this an expansion of multiple items of an array? isArrayExpansion (T_DollarBraced _ _ l) = let string = concat $ oversimplify l in @@ -297,7 +325,7 @@ getCommand t = -- Maybe get the command name string of a token representing a command getCommandName :: Token -> Maybe String -getCommandName = fst . getCommandNameAndToken +getCommandName = fst . getCommandNameAndToken False -- Maybe get the name+arguments of a command. getCommandArgv t = do @@ -307,18 +335,37 @@ getCommandArgv t = do -- Get the command name token from a command, i.e. -- the token representing 'ls' in 'ls -la 2> foo'. -- If it can't be determined, return the original token. -getCommandTokenOrThis = snd . getCommandNameAndToken +getCommandTokenOrThis = snd . getCommandNameAndToken False -getCommandNameAndToken :: Token -> (Maybe String, Token) -getCommandNameAndToken t = fromMaybe (Nothing, t) $ do - (T_SimpleCommand _ _ (w:rest)) <- getCommand t +-- Given a command, get the string and token that represents the command name. +-- If direct, return the actual command (e.g. exec in 'exec ls') +-- If not, return the logical command (e.g. 'ls' in 'exec ls') + +getCommandNameAndToken :: Bool -> Token -> (Maybe String, Token) +getCommandNameAndToken direct t = fromMaybe (Nothing, t) $ do + cmd@(T_SimpleCommand _ _ (w:rest)) <- getCommand t s <- getLiteralString w - return $ case rest of - (applet:_) | "busybox" `isSuffixOf` s || "builtin" == s -> - (getLiteralString applet, applet) - _ -> - (Just s, w) - + return $ fromMaybe (Just s, w) $ do + guard $ not direct + actual <- getEffectiveCommandToken s cmd rest + return (getLiteralString actual, actual) + where + getEffectiveCommandToken str cmd args = + let + firstArg = do + arg <- listToMaybe args + guard . not $ isFlag arg + return arg + in + case str of + "busybox" -> firstArg + "builtin" -> firstArg + "command" -> firstArg + "exec" -> do + opts <- getBsdOpts "cla:" cmd + (_, t) <- listToMaybe $ filter (null . fst) opts + return t + _ -> fail "" -- If a command substitution is a single command, get its name. -- $(date +%s) = Just "date" @@ -335,8 +382,8 @@ getCommandNameFromExpansion t = -- Get the basename of a token representing a command getCommandBasename = fmap basename . getCommandName - where - basename = reverse . takeWhile (/= '/') . reverse + +basename = reverse . takeWhile (/= '/') . reverse isAssignment t = case t of diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 977d415..4071764 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -942,8 +942,10 @@ prop_checkSingleQuotedVariables18= verifyNot checkSingleQuotedVariables "echo '` prop_checkSingleQuotedVariables19= verifyNot checkSingleQuotedVariables "echo '```'" prop_checkSingleQuotedVariables20= verifyNot checkSingleQuotedVariables "mumps -run %XCMD 'W $O(^GLOBAL(5))'" prop_checkSingleQuotedVariables21= verifyNot checkSingleQuotedVariables "mumps -run LOOP%XCMD --xec 'W $O(^GLOBAL(6))'" - - +prop_checkSingleQuotedVariables22= verifyNot checkSingleQuotedVariables "jq '$__loc__'" +prop_checkSingleQuotedVariables23= verifyNot checkSingleQuotedVariables "command jq '$__loc__'" +prop_checkSingleQuotedVariables24= verifyNot checkSingleQuotedVariables "exec jq '$__loc__'" +prop_checkSingleQuotedVariables25= verifyNot checkSingleQuotedVariables "exec -c -a foo jq '$__loc__'" checkSingleQuotedVariables params t@(T_SingleQuoted id s) = @@ -1677,13 +1679,10 @@ checkSpuriousExec _ = doLists doList tail True doList' _ _ = return () - commentIfExec (T_Pipeline id _ list) = - mapM_ commentIfExec $ take 1 list - commentIfExec (T_Redirecting _ _ f@( - T_SimpleCommand id _ (cmd:arg:_))) - | f `isUnqualifiedCommand` "exec" = - warn id 2093 - "Remove \"exec \" if script should continue after this command." + commentIfExec (T_Pipeline id _ [c]) = commentIfExec c + commentIfExec (T_Redirecting _ _ (T_SimpleCommand id _ (cmd:additionalArg:_))) | + getLiteralString cmd == Just "exec" = + warn id 2093 "Remove \"exec \" if script should continue after this command." commentIfExec _ = return () @@ -2056,18 +2055,27 @@ prop_checkFunctionsUsedExternally6 = verifyNotTree checkFunctionsUsedExternally "foo() { :; }; ssh host echo foo" prop_checkFunctionsUsedExternally7 = verifyNotTree checkFunctionsUsedExternally "install() { :; }; sudo apt-get install foo" +prop_checkFunctionsUsedExternally8 = + verifyTree checkFunctionsUsedExternally "foo() { :; }; command sudo foo" +prop_checkFunctionsUsedExternally9 = + verifyTree checkFunctionsUsedExternally "foo() { :; }; exec -c sudo foo" checkFunctionsUsedExternally params t = runNodeAnalysis checkCommand params t where - checkCommand _ t@(T_SimpleCommand _ _ (cmd:args)) = - case getCommandBasename t of - Just name -> do + checkCommand _ t@(T_SimpleCommand _ _ argv) = + case getCommandNameAndToken False t of + (Just str, t) -> do + let name = basename str + let args = skipOver t argv let argStrings = map (\x -> (fromMaybe "" $ getLiteralString x, x)) args let candidates = getPotentialCommands name argStrings mapM_ (checkArg name) candidates _ -> return () checkCommand _ _ = return () + skipOver t list = drop 1 $ dropWhile (\c -> getId c /= id) $ list + where id = getId t + -- Try to pick out the argument[s] that may be commands getPotentialCommands name argAndString = case name of diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 69ebfe6..ab1c415 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -901,33 +901,6 @@ isQuotedAlternativeReference t = where re = mkRegex "(^|\\]):?\\+" --- getGnuOpts "erd:u:" will parse a SimpleCommand like --- read -re -d : -u 3 bar --- into --- Just [("r", -re), ("e", -re), ("d", :), ("u", 3), ("", bar)] --- where flags with arguments map to arguments, while others map to themselves. --- Any unrecognized flag will result in Nothing. -getGnuOpts str t = getOpts str $ getAllFlags t -getBsdOpts str t = getOpts str $ getLeadingFlags t -getOpts :: String -> [(Token, String)] -> Maybe [(String, Token)] -getOpts string flags = process flags - where - flagList (c:':':rest) = ([c], True) : flagList rest - flagList (c:rest) = ([c], False) : flagList rest - flagList [] = [] - flagMap = Map.fromList $ ("", False) : flagList string - - process [] = return [] - process ((token1, flag):rest1) = do - takesArg <- Map.lookup flag flagMap - (token, rest) <- if takesArg - then case rest1 of - (token2, ""):rest2 -> return (token2, rest2) - _ -> fail "takesArg without valid arg" - else return (token1, rest1) - more <- process rest - return $ (flag, token) : more - supportsArrays Bash = True supportsArrays Ksh = True supportsArrays _ = False diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 1f87868..2482207 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -280,7 +280,7 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do flagRegex = mkRegex "^-[eEsn]+$" bashism t@(T_SimpleCommand _ _ (cmd:arg:_)) - | t `isCommand` "exec" && "-" `isPrefixOf` concat (oversimplify arg) = + | getLiteralString cmd == Just "exec" && "-" `isPrefixOf` concat (oversimplify arg) = warnMsg (getId arg) "exec flags are" bashism t@(T_SimpleCommand id _ _) | t `isCommand` "let" = warnMsg id "'let' is" From 14e680609271ca77b9f1d49ca7fa44787b98b4b2 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 25 Jul 2020 17:36:22 -0700 Subject: [PATCH 070/459] Handle literal linefeeds in printf format strings (fixes #2007) --- src/ShellCheck/Checks/Commands.hs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 63e58a1..b4e8b3e 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -560,6 +560,8 @@ prop_checkPrintfVar18= verifyNot checkPrintfVar "printf '%-*s\\n' 1 2" prop_checkPrintfVar19= verifyNot checkPrintfVar "printf '%(%s)T'" prop_checkPrintfVar20= verifyNot checkPrintfVar "printf '%d %(%s)T' 42" prop_checkPrintfVar21= verify checkPrintfVar "printf '%d %(%s)T'" +prop_checkPrintfVar22= verify checkPrintfVar "printf '%s\n%s' foo" + checkPrintfVar = CommandCheck (Exactly "printf") (f . arguments) where f (doubledash:rest) | getLiteralString doubledash == Just "--" = f rest f (dashv:var:rest) | getLiteralString dashv == Just "-v" = f rest @@ -602,6 +604,8 @@ prop_checkGetPrintfFormats2 = getPrintfFormats "%0*s" == "*s" prop_checkGetPrintfFormats3 = getPrintfFormats "%(%s)T" == "T" prop_checkGetPrintfFormats4 = getPrintfFormats "%d%%%(%s)T" == "dT" prop_checkGetPrintfFormats5 = getPrintfFormats "%bPassed: %d, %bFailed: %d%b, Skipped: %d, %bErrored: %d%b\\n" == "bdbdbdbdb" +prop_checkGetPrintfFormats6 = getPrintfFormats "%s%s" == "ss" +prop_checkGetPrintfFormats7 = getPrintfFormats "%s\n%s" == "ss" getPrintfFormats = getFormats where -- Get the arguments in the string as a string of type characters, @@ -620,17 +624,17 @@ getPrintfFormats = getFormats regexBasedGetFormats rest = case matchRegex re rest of - Just [width, precision, typ, rest] -> + Just [width, precision, typ, rest, _] -> (if width == "*" then "*" else "") ++ (if precision == "*" then "*" else "") ++ typ ++ getFormats rest Nothing -> take 1 rest ++ getFormats rest where -- constructed based on specifications in "man printf" - re = mkRegex "#?-?\\+? ?0?(\\*|\\d*)\\.?(\\d*|\\*)([diouxXfFeEgGaAcsbq])(.*)" - -- \____ _____/\___ ____/ \____ ____/\_________ _________/ \ / - -- V V V V V - -- flags field width precision format character rest + re = mkRegex "#?-?\\+? ?0?(\\*|\\d*)\\.?(\\d*|\\*)([diouxXfFeEgGaAcsbq])((\n|.)*)" + -- \____ _____/\___ ____/ \____ ____/\_________ _________/ \______ / + -- V V V V V + -- flags field width precision format character rest -- field width and precision can be specified with a '*' instead of a digit, -- in which case printf will accept one more argument for each '*' used From 34885142e795fccce0a3f4c19329ca2badf09c12 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 27 Jul 2020 18:34:42 -0700 Subject: [PATCH 071/459] Handle tilde expansion in pattern matching (fixes #1769) --- src/ShellCheck/ASTLib.hs | 39 +++++++++++++++++++++---------------- src/ShellCheck/Analytics.hs | 3 +++ 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 09c7537..29feb1e 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -463,30 +463,35 @@ data PseudoGlob = PGAny | PGMany | PGChar Char -- Turn a word into a PG pattern, replacing all unknown/runtime values with -- PGMany. wordToPseudoGlob :: Token -> [PseudoGlob] -wordToPseudoGlob word = - simplifyPseudoGlob . concatMap f $ getWordParts word - where - f x = case x of - T_Literal _ s -> map PGChar s - T_SingleQuoted _ s -> map PGChar s - - T_Glob _ "?" -> [PGAny] - T_Glob _ ('[':_) -> [PGAny] - - _ -> [PGMany] +wordToPseudoGlob = fromMaybe [PGMany] . wordToPseudoGlob' False -- Turn a word into a PG pattern, but only if we can preserve -- exact semantics. wordToExactPseudoGlob :: Token -> Maybe [PseudoGlob] -wordToExactPseudoGlob word = - simplifyPseudoGlob . concat <$> mapM f (getWordParts word) +wordToExactPseudoGlob = wordToPseudoGlob' True + +wordToPseudoGlob' :: Bool -> Token -> Maybe [PseudoGlob] +wordToPseudoGlob' exact word = + simplifyPseudoGlob <$> toGlob word where + toGlob :: Token -> Maybe [PseudoGlob] + toGlob word = + case word of + T_NormalWord _ (T_Literal _ ('~':str):rest) -> do + guard $ not exact + let this = (PGMany : (map PGChar $ dropWhile (/= '/') str)) + tail <- concat <$> (mapM f $ concatMap getWordParts rest) + return $ this ++ tail + _ -> concat <$> (mapM f $ getWordParts word) + f x = case x of - T_Literal _ s -> return $ map PGChar s + T_Literal _ s -> return $ map PGChar s T_SingleQuoted _ s -> return $ map PGChar s - T_Glob _ "?" -> return [PGAny] - T_Glob _ "*" -> return [PGMany] - _ -> fail "Unknown token type" + T_Glob _ "?" -> return [PGAny] + T_Glob _ "*" -> return [PGMany] + T_Glob _ ('[':_) | not exact -> return [PGAny] + _ -> if exact then fail "" else return [PGMany] + -- Reorder a PseudoGlob for more efficient matching, e.g. -- f?*?**g -> f??*g diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 4071764..bba3caf 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1211,6 +1211,9 @@ prop_checkConstantIfs6 = verifyNot checkConstantIfs "[[ a -ot b ]]" prop_checkConstantIfs7 = verifyNot checkConstantIfs "[ a -nt b ]" prop_checkConstantIfs8 = verifyNot checkConstantIfs "[[ ~foo == '~foo' ]]" prop_checkConstantIfs9 = verify checkConstantIfs "[[ *.png == [a-z] ]]" +prop_checkConstantIfs10 = verifyNot checkConstantIfs "[[ ~me == ~+ ]]" +prop_checkConstantIfs11 = verifyNot checkConstantIfs "[[ ~ == ~+ ]]" +prop_checkConstantIfs12 = verify checkConstantIfs "[[ '~' == x ]]" checkConstantIfs _ (TC_Binary id typ op lhs rhs) | not isDynamic = if isConstant lhs && isConstant rhs then warn id 2050 "This expression is constant. Did you forget the $ on a variable?" From cc81bdee31fdf6e9ff90be19f8619307db401358 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 27 Jul 2020 18:44:07 -0700 Subject: [PATCH 072/459] Improve SC1033/SC1034 message --- src/ShellCheck/Parser.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index bb27ea3..e5b10f4 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -945,8 +945,8 @@ readCondition = called "test expression" $ do cpos <- getPosition close <- try (string "]]") <|> string "]" <|> fail "Expected test to end here (don't wrap commands in []/[[]])" id <- endSpan start - when (open == "[[" && close /= "]]") $ parseProblemAt cpos ErrorC 1033 "Did you mean ]] ?" - when (open == "[" && close /= "]" ) $ parseProblemAt opos ErrorC 1034 "Did you mean [[ ?" + when (open == "[[" && close /= "]]") $ parseProblemAt cpos ErrorC 1033 "Test expression was opened with double [[ but closed with single ]. Make sure they match." + when (open == "[" && close /= "]" ) $ parseProblemAt opos ErrorC 1034 "Test expression was opened with single [ but closed with double ]]. Make sure they match." optional $ lookAhead $ do pos <- getPosition notFollowedBy2 readCmdWord <|> From 1ac2c31728dd3109aea3c120d8fa021d358319af Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 27 Jul 2020 21:28:48 -0700 Subject: [PATCH 073/459] Warn when shell functions blatantly recurse (fixes #1994) --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 37 +++++++++++++++++++++++++++++++++++ src/ShellCheck/AnalyzerLib.hs | 2 ++ 3 files changed, 40 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fde8b0f..a9ea5f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ - SC2259/SC2260: Warn when redirections override pipes - SC2261: Warn about multiple competing redirections - SC2262/SC2263: Warn about aliases declared and used in the same parsing unit +- SC2264: Warn about wrapper functions that blatantly recurse ### Fixed - SC1072/SC1073 now respond to disable annotations, though ignoring parse errors diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index bba3caf..f8904be 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -191,6 +191,7 @@ nodeChecks = [ ,checkUselessBang ,checkTranslatedStringVariable ,checkModifiedArithmeticInRedirection + ,checkBlatantRecursion ] optionalChecks = map fst optionalTreeChecks @@ -3824,5 +3825,41 @@ groupByLink f list = else (reverse $ current:span) : g next [] rest g current span [] = [reverse (current:span)] + +prop_checkBlatantRecursion1 = verify checkBlatantRecursion ":(){ :|:& };:" +prop_checkBlatantRecursion2 = verify checkBlatantRecursion "f() { f; }" +prop_checkBlatantRecursion3 = verifyNot checkBlatantRecursion "f() { command f; }" +prop_checkBlatantRecursion4 = verify checkBlatantRecursion "cd() { cd \"$lol/$1\" || exit; }" +prop_checkBlatantRecursion5 = verifyNot checkBlatantRecursion "cd() { [ -z \"$1\" ] || cd \"$1\"; }" +prop_checkBlatantRecursion6 = verifyNot checkBlatantRecursion "cd() { something; cd $1; }" +prop_checkBlatantRecursion7 = verifyNot checkBlatantRecursion "cd() { builtin cd $1; }" +checkBlatantRecursion :: Parameters -> Token -> Writer [TokenComment] () +checkBlatantRecursion params t = + case t of + T_Function _ _ _ name body -> + case getCommandSequences body of + [first : _] -> checkList name first + _ -> return () + _ -> return () + where + checkList :: String -> Token -> Writer [TokenComment] () + checkList name t = + case t of + T_Backgrounded _ t -> checkList name t + T_AndIf _ lhs _ -> checkList name lhs + T_OrIf _ lhs _ -> checkList name lhs + T_Pipeline _ _ cmds -> mapM_ (checkCommand name) cmds + _ -> return () + + checkCommand :: String -> Token -> Writer [TokenComment] () + checkCommand name cmd = sequence_ $ do + let (invokedM, t) = getCommandNameAndToken True cmd + invoked <- invokedM + guard $ name == invoked + return $ + errWithFix (getId t) 2264 + ("This function unconditionally re-invokes itself. Missing 'command'?") + (fixWith [replaceStart (getId t) params 0 $ "command "]) + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index ab1c415..1ff50b2 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -163,6 +163,8 @@ err id code str = addComment $ makeComment ErrorC id code str info id code str = addComment $ makeComment InfoC id code str style id code str = addComment $ makeComment StyleC id code str +errWithFix :: MonadWriter [TokenComment] m => Id -> Code -> String -> Fix -> m () +errWithFix = addCommentWithFix ErrorC warnWithFix :: MonadWriter [TokenComment] m => Id -> Code -> String -> Fix -> m () warnWithFix = addCommentWithFix WarningC styleWithFix :: MonadWriter [TokenComment] m => Id -> Code -> String -> Fix -> m () From beee9b22cadead5066b58c0bb67047d142f7c6f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9E=97=E5=8D=9A=E4=BB=81=28Buo-ren=2C=20Lin=29?= Date: Sun, 2 Aug 2020 16:55:45 +0800 Subject: [PATCH 074/459] Fix snap distribution unable to process scripts in Unicode(Chinese) (fixes #1643) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The snap runtime only supports C.UTF-8 locale, as other locales don't seem to be used now just hardcode it. Fixes #1643. Signed-off-by: 林博仁(Buo-ren, Lin) --- snap/snapcraft.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 8365b13..2dc4831 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -31,6 +31,8 @@ apps: shellcheck: command: usr/bin/shellcheck plugs: [home, removable-media] + environment: + LANG: C.UTF-8 parts: shellcheck: From c5b6d6f027c700a1ab075c5f84da5aa7a6ca0df8 Mon Sep 17 00:00:00 2001 From: Brennan Vincent Date: Wed, 5 Aug 2020 10:50:14 -0400 Subject: [PATCH 075/459] Update README.md Minor typographical fixes ("macOS" everywhere, capitalize Homebrew) --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index bfb3a99..6f38fa5 100644 --- a/README.md +++ b/README.md @@ -167,7 +167,7 @@ On FreeBSD: pkg install hs-ShellCheck -On OS X with homebrew: +On macOS (OS X) with Homebrew: brew install shellcheck @@ -220,7 +220,7 @@ Alternatively, you can download pre-compiled binaries for the latest release her * [Linux, x86_64](https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.x86_64.tar.xz) (statically linked) * [Linux, armv6hf](https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.armv6hf.tar.xz), i.e. Raspberry Pi (statically linked) * [Linux, aarch64](https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.aarch64.tar.xz) aka ARM64 (statically linked) -* [MacOS, x86_64](https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.darwin.x86_64.tar.xz) +* [macOS, x86_64](https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.darwin.x86_64.tar.xz) * [Windows, x86](https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.zip) or see the [GitHub Releases](https://github.com/koalaman/shellcheck/releases) for other releases @@ -264,7 +264,7 @@ This section describes how to build ShellCheck from a source directory. ShellChe ShellCheck is built and packaged using Cabal. Install the package `cabal-install` from your system's package manager (with e.g. `apt-get`, `brew`, `emerge`, `yum`, or `zypper`). -On MacOS (OS X), you can do a fast install of Cabal using brew, which takes a couple of minutes instead of more than 30 minutes if you try to compile it from source. +On macOS (OS X), you can do a fast install of Cabal using brew, which takes a couple of minutes instead of more than 30 minutes if you try to compile it from source. $ brew install cabal-install From 3e50a2fce8323f89b63243d2b9042d756d327920 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Fri, 7 Aug 2020 14:59:34 -0700 Subject: [PATCH 076/459] Suppress SC2216 for du --files0-from or --exclude-from (fixes #1286) --- src/ShellCheck/Analytics.hs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index f8904be..7906e7a 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -3305,6 +3305,8 @@ prop_checkPipeToNowhere15 = verifyNot checkPipeToNowhere "ls > foo 2> bar |& gre prop_checkPipeToNowhere16 = verifyNot checkPipeToNowhere "echo World | cat << EOF\nhello $(cat)\nEOF\n" prop_checkPipeToNowhere17 = verify checkPipeToNowhere "echo World | cat << 'EOF'\nhello $(cat)\nEOF\n" prop_checkPipeToNowhere18 = verifyNot checkPipeToNowhere "ls 1>&3 3>&1 3>&- | wc -l" +prop_checkPipeToNowhere19 = verifyNot checkPipeToNowhere "find . -print0 | du --files0-from=/dev/stdin" +prop_checkPipeToNowhere20 = verifyNot checkPipeToNowhere "find . | du --exclude-from=/dev/fd/0" data PipeType = StdoutPipe | StdoutStderrPipe | NoPipe deriving (Eq) checkPipeToNowhere :: Parameters -> Token -> WriterT [TokenComment] Identity () @@ -3324,6 +3326,7 @@ checkPipeToNowhere params t = name <- getCommandBasename cmd guard $ name `elem` nonReadingCommands guard $ not hasConsumers && input /= NoPipe + guard . not $ commandSpecificException name cmd -- Confusing echo for cat is so common that it's worth a special case let suggestion = @@ -3366,6 +3369,11 @@ checkPipeToNowhere params t = outputWarning mapM_ warnAboutDupes $ Map.assocs fdMap + commandSpecificException name cmd = + case name of + "du" -> any (`elem` ["exclude-from", "files0-from"]) $ lt $ map snd $ getAllFlags cmd + _ -> False + warnAboutDupes (n, list@(_:_:_)) = forM_ list $ \c -> err (getOpId c) 2261 $ "Multiple redirections compete for " ++ str n ++ ". Use cat, tee, or pass filenames instead." From e779aedac3355bba0c6d071c741845d62d7aa4d8 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Fri, 7 Aug 2020 16:00:58 -0700 Subject: [PATCH 077/459] Modernize getting mapfile array name --- src/ShellCheck/Analytics.hs | 4 +++- src/ShellCheck/AnalyzerLib.hs | 35 ++++++++++++++++++++++++----------- 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 9ad53ae..f5c1a18 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2228,7 +2228,9 @@ prop_checkUnassignedReferences37= verifyNotTree checkUnassignedReferences "var=h prop_checkUnassignedReferences38= verifyTree (checkUnassignedReferences' True) "echo $VAR" prop_checkUnassignedReferences39= verifyNotTree checkUnassignedReferences "builtin export var=4; echo $var" prop_checkUnassignedReferences40= verifyNotTree checkUnassignedReferences ": ${foo=bar}" -prop_checkUnassignedReferences41= verifyNotTree checkUnassignedReferences "mapfile -t files <(cat); echo \"${files[@]}\"" +prop_checkUnassignedReferences41= verifyNotTree checkUnassignedReferences "mapfile -t files 123; echo \"${files[@]}\"" +prop_checkUnassignedReferences42= verifyNotTree checkUnassignedReferences "mapfile files -t; echo \"${files[@]}\"" +prop_checkUnassignedReferences43= verifyNotTree checkUnassignedReferences "mapfile --future files; echo \"${files[@]}\"" checkUnassignedReferences = checkUnassignedReferences' False checkUnassignedReferences' includeGlobals params t = warnings diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index ba2c901..596340b 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -658,17 +658,30 @@ getModifiedVariableCommand base@(T_SimpleCommand id cmdPrefix (T_NormalWord _ (T f [] = fail "not found" -- mapfile has some curious syntax allowing flags plus 0..n variable names - -- where only the first non-option one is used if any. Here we cheat and - -- just get the last one, if it's a variable name, and omitting process - -- substitions. - getMapfileArray base arguments = do - lastArg <- listToMaybe (filter notProcSub $ reverse arguments) - name <- getLiteralString lastArg - guard $ isVariableName name - return (base, lastArg, name, DataArray SourceExternal) - - notProcSub (T_NormalWord _ (T_ProcSub{} :_)) = False - notProcSub _ = True + -- where only the first non-option one is used if any. + getMapfileArray base rest = parseArgs `mplus` fallback + where + parseArgs :: Maybe (Token, Token, String, DataType) + parseArgs = do + args <- getGnuOpts "d:n:O:s:u:C:c:t" base + let names = map snd $ filter (\(x,y) -> null x) args + if null names + then + return (base, base, "MAPFILE", DataArray SourceExternal) + else do + first <- listToMaybe names + name <- getLiteralString first + guard $ isVariableName name + return (base, first, name, DataArray SourceExternal) + -- If arg parsing fails (due to bad or new flags), get the last variable name + fallback :: Maybe (Token, Token, String, DataType) + fallback = do + (name, token) <- listToMaybe . mapMaybe f $ reverse rest + return (base, token, name, DataArray SourceExternal) + f arg = do + name <- getLiteralString arg + guard $ isVariableName name + return (name, arg) -- get all the array variables used in read, e.g. read -a arr getReadArrayVariables args = From 50067ddf94c04b2835a44a0ec16e3772e0a01835 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 8 Aug 2020 12:32:20 -0700 Subject: [PATCH 078/459] Consider variables in -z/-n tests to be checked --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 4 ---- src/ShellCheck/AnalyzerLib.hs | 21 ++++++++++++--------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a9ea5f4..1b3a2f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ ### Changed - SC1090: A leading `$x/` or `$(x)/` is now treated as `./` when locating files +- SC2154: Variables appearing in -z/-n tests are no longer considered unassigned ## v0.7.1 - 2020-04-04 diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 40f6375..9b92698 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2231,7 +2231,6 @@ prop_checkUnassignedReferences40= verifyNotTree checkUnassignedReferences ": ${f prop_checkUnassignedReferences41= verifyNotTree checkUnassignedReferences "mapfile -t files 123; echo \"${files[@]}\"" prop_checkUnassignedReferences42= verifyNotTree checkUnassignedReferences "mapfile files -t; echo \"${files[@]}\"" prop_checkUnassignedReferences43= verifyNotTree checkUnassignedReferences "mapfile --future files; echo \"${files[@]}\"" - prop_checkUnassignedReferences_minusNPlain = verifyNotTree checkUnassignedReferences "if [ -n \"$x\" ]; then echo $x; fi" prop_checkUnassignedReferences_minusZPlain = verifyNotTree checkUnassignedReferences "if [ -z \"$x\" ]; then echo \"\"; fi" prop_checkUnassignedReferences_minusNBraced = verifyNotTree checkUnassignedReferences "if [ -n \"${x}\" ]; then echo $x; fi" @@ -2239,9 +2238,6 @@ prop_checkUnassignedReferences_minusZBraced = verifyNotTree checkUnassignedRefe prop_checkUnassignedReferences_minusNDefault = verifyNotTree checkUnassignedReferences "if [ -n \"${x:-}\" ]; then echo $x; fi" prop_checkUnassignedReferences_minusZDefault = verifyNotTree checkUnassignedReferences "if [ -z \"${x:-}\" ]; then echo \"\"; fi" -prop_checkUnassignedReferences_minusZInsteadOfN = verifyTree checkUnassignedReferences "if [ -z \"$x\" ]; then echo $x; fi" -prop_checkUnassignedReferences_minusZInsteadOfNBraced = verifyTree checkUnassignedReferences "if [ -z \"${x}\" ]; then echo $x; fi" - checkUnassignedReferences = checkUnassignedReferences' False checkUnassignedReferences' includeGlobals params t = warnings where diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index d87362c..c9954d1 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -499,8 +499,9 @@ getModifiedVariables t = guard . not . null $ str return (t, token, str, DataString SourceChecked) - TC_Unary _ _ "-n" (T_NormalWord _ [T_DoubleQuoted _ [db@(T_DollarBraced _ _ l)]]) -> - [(t, t, getBracedReference (concat $ oversimplify l), DataString SourceChecked)] + TC_Unary _ _ "-n" token -> markAsChecked t token + TC_Unary _ _ "-z" token -> markAsChecked t token + TC_Nullary _ _ token -> markAsChecked t token T_DollarBraced _ _ l -> maybeToList $ do let string = concat $ oversimplify l @@ -519,6 +520,14 @@ getModifiedVariables t = T_ForIn id str words _ -> [(t, t, str, DataString $ SourceFrom words)] T_SelectIn id str words _ -> [(t, t, str, DataString $ SourceFrom words)] _ -> [] + where + markAsChecked place token = mapMaybe (f place) $ getWordParts token + f place t = case t of + T_DollarBraced _ _ l -> + let str = getBracedReference $ concat $ oversimplify l in do + guard $ isVariableName str + return (place, t, str, DataString SourceChecked) + _ -> Nothing isClosingFileOp op = case op of @@ -726,9 +735,7 @@ getOffsetReferences mods = fromMaybe [] $ do getReferencedVariables parents t = case t of T_DollarBraced id _ l -> let str = concat $ oversimplify l in - if isMinusZTest t - then [] - else (t, t, getBracedReference str) : + (t, t, getBracedReference str) : map (\x -> (l, l, x)) ( getIndexReferences str ++ getOffsetReferences (getBracedModifier str)) @@ -782,10 +789,6 @@ getReferencedVariables parents t = this: TA_Assignment _ "=" lhs _ :_ -> lhs == t _ -> False - isMinusZTest t = case getPath parents t of - _ : T_DoubleQuoted _ [_] : T_NormalWord _ [_] : TC_Unary _ SingleBracket "-z" _ : _ -> True - _ -> False - dataTypeFrom defaultType v = (case v of T_Array {} -> DataArray; _ -> defaultType) $ SourceFrom [v] From e72fbb2640a9aee77d1e380c6f9364fd2b500bb9 Mon Sep 17 00:00:00 2001 From: Simon Shine Date: Thu, 20 Aug 2020 13:07:32 +0200 Subject: [PATCH 079/459] Fix whitespace in README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e400651..05ce5ad 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ Services and platforms that have ShellCheck pre-installed and ready to use: * [Code Climate](https://codeclimate.com/) * [Code Factor](https://www.codefactor.io/) * [CircleCI](https://circleci.com) via the [ShellCheck Orb](https://circleci.com/orbs/registry/orb/circleci/shellcheck) -* [Github](https://github.com/features/actions)(only Linux) +* [Github](https://github.com/features/actions) (only Linux) Services and platforms with third party plugins: From a62d9f10c29587056cdad30942b984c3e492dfc1 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 23 Aug 2020 15:43:33 -0700 Subject: [PATCH 080/459] Warn when using &/| between test statements --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 46 +++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b3a2f0..3d91611 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - SC2261: Warn about multiple competing redirections - SC2262/SC2263: Warn about aliases declared and used in the same parsing unit - SC2264: Warn about wrapper functions that blatantly recurse +- SC2265/SC2266: Warn when using & or | with test statements ### Fixed - SC1072/SC1073 now respond to disable annotations, though ignoring parse errors diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 9b92698..ffcf9cf 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -192,6 +192,7 @@ nodeChecks = [ ,checkTranslatedStringVariable ,checkModifiedArithmeticInRedirection ,checkBlatantRecursion + ,checkBadTestAndOr ] optionalChecks = map fst optionalTreeChecks @@ -3878,5 +3879,50 @@ checkBlatantRecursion params t = ("This function unconditionally re-invokes itself. Missing 'command'?") (fixWith [replaceStart (getId t) params 0 $ "command "]) + +prop_checkBadTestAndOr1 = verify checkBadTestAndOr "[ x ] & [ y ]" +prop_checkBadTestAndOr2 = verify checkBadTestAndOr "test -e foo & [ y ]" +prop_checkBadTestAndOr3 = verify checkBadTestAndOr "[ x ] | [ y ]" +checkBadTestAndOr params t = + case t of + T_Pipeline _ seps cmds@(_:_:_) -> checkOrs seps cmds + T_Backgrounded id cmd -> checkAnds id cmd + _ -> return () + where + checkOrs seps cmds = + let maybeSeps = map Just seps + commandWithSeps = zip3 (Nothing:maybeSeps) cmds (maybeSeps ++ [Nothing]) + in + mapM_ checkTest commandWithSeps + checkTest (before, cmd, after) = + when (isTest cmd) $ do + checkPipe before + checkPipe after + + checkPipe t = + case t of + Just (T_Pipe id "|") -> + warnWithFix id 2266 "Use || for logical OR. Single | will pipe." $ + fixWith [replaceEnd id params 0 "|"] + _ -> return () + + checkAnds id t = + case t of + T_AndIf _ _ rhs -> checkAnds id rhs + T_OrIf _ _ rhs -> checkAnds id rhs + T_Pipeline _ _ list | not (null list) -> checkAnds id (last list) + cmd -> when (isTest cmd) $ + errWithFix id 2265 "Use && for logical AND. Single & will background and return true." $ + (fixWith [replaceEnd id params 0 "&"]) + + isTest t = + case t of + T_Condition {} -> True + T_SimpleCommand {} -> t `isCommand` "test" + T_Redirecting _ _ t -> isTest t + T_Annotation _ _ t -> isTest t + _ -> False + + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) From 9e59bcca9131359b9f418b209cbfbe41116a3679 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 23 Aug 2020 15:49:20 -0700 Subject: [PATCH 081/459] Upgrade SC2169 (unsupported in dash) from warning to error (fixes #2013) --- src/ShellCheck/Checks/ShellSupport.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 2482207..788323d 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -190,7 +190,7 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do isDash = shellType params == Dash warnMsg id s = if isDash - then warn id 2169 $ "In dash, " ++ s ++ " not supported." + then err id 2169 $ "In dash, " ++ s ++ " not supported." else warn id 2039 $ "In POSIX sh, " ++ s ++ " undefined." bashism (T_ProcSub id _ _) = warnMsg id "process substitution is" From c9be7ab2eb7ccb05ca0a5cb75946469a3cee98e0 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 23 Aug 2020 18:46:13 -0700 Subject: [PATCH 082/459] Parse assignments according to spec (fixes #2022) --- CHANGELOG.md | 1 + src/ShellCheck/Parser.hs | 95 +++++++++++++++++++++++++--------------- 2 files changed, 60 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d91611..5db445b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ is still purely cosmetic and does not allow ShellCheck to continue. ### Changed +- Assignments are now parsed to spec, without leniency for leading $ or spaces - SC1090: A leading `$x/` or `$(x)/` is now treated as `./` when locating files - SC2154: Variables appearing in -z/-n tests are no longer considered unassigned diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index e5b10f4..4806b18 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -212,6 +212,12 @@ endSpan (IncompleteInterval start) = do id <- getNextIdBetween start endPos return id +getSpanPositionsFor m = do + start <- getPosition + m + end <- getPosition + return (start, end) + addToHereDocMap id list = do state <- getState let map = hereDocMap state @@ -2788,17 +2794,13 @@ readLiteralForParser parser = do prop_readAssignmentWord = isOk readAssignmentWord "a=42" prop_readAssignmentWord2 = isOk readAssignmentWord "b=(1 2 3)" -prop_readAssignmentWord3 = isWarning readAssignmentWord "$b = 13" -prop_readAssignmentWord4 = isWarning readAssignmentWord "b = $(lol)" prop_readAssignmentWord5 = isOk readAssignmentWord "b+=lol" -prop_readAssignmentWord6 = isWarning readAssignmentWord "b += (1 2 3)" prop_readAssignmentWord7 = isOk readAssignmentWord "a[3$n'']=42" prop_readAssignmentWord8 = isOk readAssignmentWord "a[4''$(cat foo)]=42" prop_readAssignmentWord9 = isOk readAssignmentWord "IFS= " prop_readAssignmentWord9a= isOk readAssignmentWord "foo=" prop_readAssignmentWord9b= isOk readAssignmentWord "foo= " prop_readAssignmentWord9c= isOk readAssignmentWord "foo= #bar" -prop_readAssignmentWord10= isWarning readAssignmentWord "foo$n=42" prop_readAssignmentWord11= isOk readAssignmentWord "foo=([a]=b [c] [d]= [e f )" prop_readAssignmentWord12= isOk readAssignmentWord "a[b <<= 3 + c]='thing'" prop_readAssignmentWord13= isOk readAssignmentWord "var=( (1 2) (3 4) )" @@ -2806,52 +2808,73 @@ prop_readAssignmentWord14= isOk readAssignmentWord "var=( 1 [2]=(3 4) )" prop_readAssignmentWord15= isOk readAssignmentWord "var=(1 [2]=(3 4))" readAssignmentWord = readAssignmentWordExt True readWellFormedAssignment = readAssignmentWordExt False -readAssignmentWordExt lenient = try $ do - start <- startSpan - pos <- getPosition - when lenient $ - optional (char '$' >> parseNote ErrorC 1066 "Don't use $ on the left side of assignments.") - variable <- readVariableName - when lenient $ - optional (readNormalDollar >> parseNoteAt pos ErrorC - 1067 "For indirection, use arrays, declare \"var$n=value\", or (for sh) read/eval.") - indices <- many readArrayIndex - hasLeftSpace <- fmap (not . null) spacing - pos <- getPosition - id <- endSpan start - op <- readAssignmentOp +readAssignmentWordExt lenient = called "variable assignment" $ do + -- Parse up to and including the = in a 'try' + (id, variable, op, indices) <- try $ do + start <- startSpan + pos <- getPosition + leadingDollarPos <- + if lenient + then optionMaybe $ getSpanPositionsFor (char '$') + else return Nothing + variable <- readVariableName + middleDollarPos <- + if lenient + then optionMaybe $ getSpanPositionsFor readNormalDollar + else return Nothing + indices <- many readArrayIndex + hasLeftSpace <- fmap (not . null) spacing + opStart <- getPosition + id <- endSpan start + op <- readAssignmentOp + opEnd <- getPosition + + when (isJust leadingDollarPos || isJust middleDollarPos || hasLeftSpace) $ do + sequence_ $ do + (l, r) <- leadingDollarPos + return $ parseProblemAtWithEnd l r ErrorC 1066 "Don't use $ on the left side of assignments." + sequence_ $ do + (l, r) <- middleDollarPos + return $ parseProblemAtWithEnd l r ErrorC 1067 "For indirection, use arrays, declare \"var$n=value\", or (for sh) read/eval." + when hasLeftSpace $ do + parseProblemAtWithEnd opStart opEnd ErrorC 1068 $ + "Don't put spaces around the " + ++ (if op == Append + then "+= when appending" + else "= in assignments") + ++ " (or quote to make it literal)." + + -- Fail so that this is not parsed as an assignment. + fail "" + -- At this point we know for sure. + return (id, variable, op, indices) + + rightPosStart <- getPosition hasRightSpace <- fmap (not . null) spacing + rightPosEnd <- getPosition isEndOfCommand <- fmap isJust $ optionMaybe (try . lookAhead $ (void (oneOf "\r\n;&|)") <|> eof)) - if not hasLeftSpace && (hasRightSpace || isEndOfCommand) + + if hasRightSpace || isEndOfCommand then do - when (variable /= "IFS" && hasRightSpace && not isEndOfCommand) $ - parseNoteAt pos WarningC 1007 + when (variable /= "IFS" && hasRightSpace && not isEndOfCommand) $ do + parseProblemAtWithEnd rightPosStart rightPosEnd WarningC 1007 "Remove space after = if trying to assign a value (for empty string, use var='' ... )." value <- readEmptyLiteral return $ T_Assignment id op variable indices value else do - when (hasLeftSpace || hasRightSpace) $ - parseNoteAt pos ErrorC 1068 $ - "Don't put spaces around the " - ++ (if op == Append - then "+= when appending" - else "= in assignments") - ++ " (or quote to make it literal)." + optional $ do + lookAhead $ char '=' + parseProblem ErrorC 1097 "Unexpected ==. For assignment, use =. For comparison, use [/[[. Or quote for literal string." + value <- readArray <|> readNormalWord spacing return $ T_Assignment id op variable indices value where readAssignmentOp = do - pos <- getPosition - unexpecting "" $ string "===" + -- This is probably some kind of ascii art border + unexpecting "===" (string "===") choice [ string "+=" >> return Append, - do - try (string "==") - parseProblemAt pos ErrorC 1097 - "Unexpected ==. For assignment, use =. For comparison, use [/[[." - return Assign, - string "=" >> return Assign ] From 43191fa71de1a67e04298db2cda14169e43d50c2 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 1 Sep 2020 14:19:28 -0700 Subject: [PATCH 083/459] Suppress SC2035 for echo * and printf * (fixes #2036) --- src/ShellCheck/Analytics.hs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index ffcf9cf..e0d33bb 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2320,8 +2320,11 @@ prop_checkGlobsAsOptions1 = verify checkGlobsAsOptions "rm *.txt" prop_checkGlobsAsOptions2 = verify checkGlobsAsOptions "ls ??.*" prop_checkGlobsAsOptions3 = verifyNot checkGlobsAsOptions "rm -- *.txt" prop_checkGlobsAsOptions4 = verifyNot checkGlobsAsOptions "*.txt" -checkGlobsAsOptions _ (T_SimpleCommand _ _ args) = - mapM_ check $ takeWhile (not . isEndOfArgs) (drop 1 args) +prop_checkGlobsAsOptions5 = verifyNot checkGlobsAsOptions "echo 'Files:' *.txt" +prop_checkGlobsAsOptions6 = verifyNot checkGlobsAsOptions "printf '%s\\n' *" +checkGlobsAsOptions _ cmd@(T_SimpleCommand _ _ args) = + unless ((fromMaybe "" $ getCommandBasename cmd) `elem` ["echo", "printf"]) $ + mapM_ check $ takeWhile (not . isEndOfArgs) (drop 1 args) where check v@(T_NormalWord _ (T_Glob id s:_)) | s == "*" || s == "?" = info id 2035 "Use ./*glob* or -- *glob* so names with dashes won't become options." From 58783ab3cc11ff30466f185a88b1ebb2f1bf19f5 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 1 Sep 2020 16:22:15 -0700 Subject: [PATCH 084/459] Allow specifying ranges in disable directives --- CHANGELOG.md | 1 + shellcheck.1.md | 3 ++- src/ShellCheck/AST.hs | 2 +- src/ShellCheck/ASTLib.hs | 2 +- src/ShellCheck/Parser.hs | 11 ++++++++--- 5 files changed, 13 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5db445b..53f63da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ## Git ### Added +- `disable` directives can now be a range, e.g. `disable=SC3000-SC4000` - SC2259/SC2260: Warn when redirections override pipes - SC2261: Warn about multiple competing redirections - SC2262/SC2263: Warn about aliases declared and used in the same parsing unit diff --git a/shellcheck.1.md b/shellcheck.1.md index 6d2a732..d038df2 100644 --- a/shellcheck.1.md +++ b/shellcheck.1.md @@ -232,7 +232,8 @@ Valid keys are: **disable** : Disables a comma separated list of error codes for the following command. The command can be a simple command like `echo foo`, or a compound command - like a function definition, subshell block or loop. + like a function definition, subshell block or loop. A range can be + be specified with a dash, e.g. `disable=SC3000-SC4000` to exclude 3xxx. **enable** : Enable an optional check by name, as listed with **--list-optional**. diff --git a/src/ShellCheck/AST.hs b/src/ShellCheck/AST.hs index d4151fe..52b6e17 100644 --- a/src/ShellCheck/AST.hs +++ b/src/ShellCheck/AST.hs @@ -145,7 +145,7 @@ data InnerToken t = deriving (Show, Eq, Functor, Foldable, Traversable) data Annotation = - DisableComment Integer + DisableComment Integer Integer -- [from, to) | EnableComment String | SourceOverride String | ShellOverride String diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 29feb1e..0052ef5 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -567,5 +567,5 @@ isAnnotationIgnoringCode code t = T_Annotation _ anns _ -> any hasNum anns _ -> False where - hasNum (DisableComment ts) = code == ts + hasNum (DisableComment from to) = code >= from && code < to hasNum _ = False diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 4806b18..87efa25 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -270,7 +270,7 @@ contextItemDisablesCode alsoCheckSourced code = disabling alsoCheckSourced ContextAnnotation list -> any disabling' list ContextSource _ -> not $ checkSourced _ -> False - disabling' (DisableComment n) = code == n + disabling' (DisableComment n m) = code >= n && code < m disabling' _ = False @@ -973,6 +973,7 @@ prop_readAnnotation3 = isOk readAnnotation "# shellcheck disable=SC1234 source=/ prop_readAnnotation4 = isWarning readAnnotation "# shellcheck cats=dogs disable=SC1234\n" prop_readAnnotation5 = isOk readAnnotation "# shellcheck disable=SC2002 # All cats are precious\n" prop_readAnnotation6 = isOk readAnnotation "# shellcheck disable=SC1234 # shellcheck foo=bar\n" +prop_readAnnotation7 = isOk readAnnotation "# shellcheck disable=SC1000,SC2000-SC3000,SC1001\n" readAnnotation = called "shellcheck directive" $ do try readAnnotationPrefix many1 linewhitespace @@ -993,12 +994,16 @@ readAnnotationWithoutPrefix = do key <- many1 (letter <|> char '-') char '=' <|> fail "Expected '=' after directive key" annotations <- case key of - "disable" -> readCode `sepBy` char ',' + "disable" -> readRange `sepBy` char ',' where + readRange = do + from <- readCode + to <- choice [ char '-' *> readCode, return $ from+1 ] + return $ DisableComment from to readCode = do optional $ string "SC" int <- many1 digit - return $ DisableComment (read int) + return $ read int "enable" -> readName `sepBy` char ',' where From cfd68ee0c2ebfd0ab08a1d4bf628162b454dc207 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 1 Sep 2020 16:48:14 -0700 Subject: [PATCH 085/459] Give each sh/dash compatibility warning its own SC3xxx error code --- CHANGELOG.md | 1 + nextnumber | 2 +- src/ShellCheck/Checker.hs | 2 +- src/ShellCheck/Checks/ShellSupport.hs | 115 +++++++++++++------------- 4 files changed, 62 insertions(+), 58 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 53f63da..d4b9145 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ ### Changed - Assignments are now parsed to spec, without leniency for leading $ or spaces +- POSIX/dash unsupported feature warnings now have individual SC3xxx codes - SC1090: A leading `$x/` or `$(x)/` is now treated as `./` when locating files - SC2154: Variables appearing in -z/-n tests are no longer considered unassigned diff --git a/nextnumber b/nextnumber index 0c9c7b1..3bdf10a 100755 --- a/nextnumber +++ b/nextnumber @@ -6,7 +6,7 @@ then exit 1 fi -for i in 1 2 +for i in 1 2 3 do last=$(grep -hv "^prop" ./**/*.hs | grep -Ewo "${i}[0-9]{3}" | sort -n | tail -n 1) echo "Next ${i}xxx: $((last+1))" diff --git a/src/ShellCheck/Checker.hs b/src/ShellCheck/Checker.hs index 673a116..d81d664 100644 --- a/src/ShellCheck/Checker.hs +++ b/src/ShellCheck/Checker.hs @@ -276,7 +276,7 @@ prop_filewideAnnotation8 = null $ check "# Disable $? warning\n#shellcheck disable=SC2181\n# Disable quoting warning\n#shellcheck disable=2086\ntrue\n[ $? == 0 ] && echo $1" prop_sourcePartOfOriginalScript = -- #1181: -x disabled posix warning for 'source' - 2039 `elem` checkWithIncludes [("./saywhat.sh", "echo foo")] "#!/bin/sh\nsource ./saywhat.sh" + 3046 `elem` checkWithIncludes [("./saywhat.sh", "echo foo")] "#!/bin/sh\nsource ./saywhat.sh" prop_spinBug1413 = null $ check "fun() {\n# shellcheck disable=SC2188\n> /dev/null\n}\n" diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 788323d..8b8e095 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -188,102 +188,102 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do kludge params = bashism where isDash = shellType params == Dash - warnMsg id s = + warnMsg id code s = if isDash - then err id 2169 $ "In dash, " ++ s ++ " not supported." - else warn id 2039 $ "In POSIX sh, " ++ s ++ " undefined." + then err id code $ "In dash, " ++ s ++ " not supported." + else warn id code $ "In POSIX sh, " ++ s ++ " undefined." - bashism (T_ProcSub id _ _) = warnMsg id "process substitution is" - bashism (T_Extglob id _ _) = warnMsg id "extglob is" - bashism (T_DollarSingleQuoted id _) = warnMsg id "$'..' is" - bashism (T_DollarDoubleQuoted id _) = warnMsg id "$\"..\" is" - bashism (T_ForArithmetic id _ _ _ _) = warnMsg id "arithmetic for loops are" - bashism (T_Arithmetic id _) = warnMsg id "standalone ((..)) is" - bashism (T_DollarBracket id _) = warnMsg id "$[..] in place of $((..)) is" - bashism (T_SelectIn id _ _ _) = warnMsg id "select loops are" - bashism (T_BraceExpansion id _) = warnMsg id "brace expansion is" - bashism (T_Condition id DoubleBracket _) = warnMsg id "[[ ]] is" - bashism (T_HereString id _) = warnMsg id "here-strings are" + bashism (T_ProcSub id _ _) = warnMsg id 3001 "process substitution is" + bashism (T_Extglob id _ _) = warnMsg id 3002 "extglob is" + bashism (T_DollarSingleQuoted id _) = warnMsg id 3003 "$'..' is" + bashism (T_DollarDoubleQuoted id _) = warnMsg id 3004 "$\"..\" is" + bashism (T_ForArithmetic id _ _ _ _) = warnMsg id 3005 "arithmetic for loops are" + bashism (T_Arithmetic id _) = warnMsg id 3006 "standalone ((..)) is" + bashism (T_DollarBracket id _) = warnMsg id 3007 "$[..] in place of $((..)) is" + bashism (T_SelectIn id _ _ _) = warnMsg id 3008 "select loops are" + bashism (T_BraceExpansion id _) = warnMsg id 3009 "brace expansion is" + bashism (T_Condition id DoubleBracket _) = warnMsg id 3010 "[[ ]] is" + bashism (T_HereString id _) = warnMsg id 3011 "here-strings are" bashism (TC_Binary id SingleBracket op _ _) | op `elem` [ "<", ">", "\\<", "\\>", "<=", ">=", "\\<=", "\\>="] = - unless isDash $ warnMsg id $ "lexicographical " ++ op ++ " is" + unless isDash $ warnMsg id 3012 $ "lexicographical " ++ op ++ " is" bashism (TC_Binary id SingleBracket op _ _) | op `elem` [ "-ot", "-nt", "-ef" ] = - unless isDash $ warnMsg id $ op ++ " is" + unless isDash $ warnMsg id 3013 $ op ++ " is" bashism (TC_Binary id SingleBracket "==" _ _) = - warnMsg id "== in place of = is" + warnMsg id 3014 "== in place of = is" bashism (TC_Binary id SingleBracket "=~" _ _) = - warnMsg id "=~ regex matching is" + warnMsg id 3015 "=~ regex matching is" bashism (TC_Unary id SingleBracket "-v" _) = - warnMsg id "unary -v (in place of [ -n \"${var+x}\" ]) is" + warnMsg id 3016 "unary -v (in place of [ -n \"${var+x}\" ]) is" bashism (TC_Unary id _ "-a" _) = - warnMsg id "unary -a in place of -e is" + warnMsg id 3017 "unary -a in place of -e is" bashism (TA_Unary id op _) | op `elem` [ "|++", "|--", "++|", "--|"] = - warnMsg id $ filter (/= '|') op ++ " is" - bashism (TA_Binary id "**" _ _) = warnMsg id "exponentials are" - bashism (T_FdRedirect id "&" (T_IoFile _ (T_Greater _) _)) = warnMsg id "&> is" - bashism (T_FdRedirect id "" (T_IoFile _ (T_GREATAND _) _)) = warnMsg id ">& is" - bashism (T_FdRedirect id ('{':_) _) = warnMsg id "named file descriptors are" + warnMsg id 3018 $ filter (/= '|') op ++ " is" + bashism (TA_Binary id "**" _ _) = warnMsg id 3019 "exponentials are" + bashism (T_FdRedirect id "&" (T_IoFile _ (T_Greater _) _)) = warnMsg id 3020 "&> is" + bashism (T_FdRedirect id "" (T_IoFile _ (T_GREATAND _) _)) = warnMsg id 3021 ">& is" + bashism (T_FdRedirect id ('{':_) _) = warnMsg id 3022 "named file descriptors are" bashism (T_FdRedirect id num _) - | all isDigit num && length num > 1 = warnMsg id "FDs outside 0-9 are" + | all isDigit num && length num > 1 = warnMsg id 3023 "FDs outside 0-9 are" bashism (T_Assignment id Append _ _ _) = - warnMsg id "+= is" + warnMsg id 3024 "+= is" bashism (T_IoFile id _ word) | isNetworked = - warnMsg id "/dev/{tcp,udp} is" + warnMsg id 3025 "/dev/{tcp,udp} is" where file = onlyLiteralString word isNetworked = any (`isPrefixOf` file) ["/dev/tcp", "/dev/udp"] bashism (T_Glob id str) | "[^" `isInfixOf` str = - warnMsg id "^ in place of ! in glob bracket expressions is" + warnMsg id 3026 "^ in place of ! in glob bracket expressions is" bashism t@(TA_Variable id str _) | isBashVariable str = - warnMsg id $ str ++ " is" + warnMsg id 3027 $ str ++ " is" bashism t@(T_DollarBraced id _ token) = do mapM_ check expansion when (isBashVariable var) $ - warnMsg id $ var ++ " is" + warnMsg id 3028 $ var ++ " is" where str = concat $ oversimplify token var = getBracedReference str check (regex, feature) = - when (isJust $ matchRegex regex str) $ warnMsg id feature + when (isJust $ matchRegex regex str) $ warnMsg id 3053 feature bashism t@(T_Pipe id "|&") = - warnMsg id "|& in place of 2>&1 | is" + warnMsg id 3029 "|& in place of 2>&1 | is" bashism (T_Array id _) = - warnMsg id "arrays are" + warnMsg id 3030 "arrays are" bashism (T_IoFile id _ t) | isGlob t = - warnMsg id "redirecting to/from globs is" + warnMsg id 3031 "redirecting to/from globs is" bashism (T_CoProc id _ _) = - warnMsg id "coproc is" + warnMsg id 3032 "coproc is" bashism (T_Function id _ _ str _) | not (isVariableName str) = - warnMsg id "naming functions outside [a-zA-Z_][a-zA-Z0-9_]* is" + warnMsg id 3033 "naming functions outside [a-zA-Z_][a-zA-Z0-9_]* is" bashism (T_DollarExpansion id [x]) | isOnlyRedirection x = - warnMsg id "$( do checkOptions (flag@(fid,flag') : opt@(oid,opt') : rest) | flag' `matches` oFlagRegex = do when (opt' `notElem` longOptions) $ - warnMsg oid $ "set option " <> opt' <> " is" + warnMsg oid 3040 $ "set option " <> opt' <> " is" checkFlags (flag:rest) | otherwise = checkFlags (flag:opt:rest) checkOptions (flag:rest) = checkFlags (flag:rest) @@ -314,10 +314,10 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do unless (flag' `matches` validFlagsRegex) $ forM_ (tail flag') $ \letter -> when (letter `notElem` optionsSet) $ - warnMsg fid $ "set flag " <> ('-':letter:" is") + warnMsg fid 3041 $ "set flag " <> ('-':letter:" is") checkOptions rest | beginsWithDoubleDash flag' = do - warnMsg fid $ "set flag " <> flag' <> " is" + warnMsg fid 3042 $ "set flag " <> flag' <> " is" checkOptions rest -- Either a word that doesn't start with a dash, or simply '--', -- so stop checking. @@ -339,16 +339,19 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do let name = fromMaybe "" $ getCommandName t flags = getLeadingFlags t in do + when (name == "local" && not isDash) $ + -- This is so commonly accepted that we'll make it a special case + warnMsg id 3043 $ "'local' is" when (name `elem` unsupportedCommands) $ - warnMsg id $ "'" ++ name ++ "' is" + warnMsg id 3044 $ "'" ++ name ++ "' is" sequence_ $ do allowed' <- Map.lookup name allowedFlags allowed <- allowed' (word, flag) <- find (\x -> (not . null . snd $ x) && snd x `notElem` allowed) flags - return . warnMsg (getId word) $ name ++ " -" ++ flag ++ " is" + return . warnMsg (getId word) 3045 $ name ++ " -" ++ flag ++ " is" - when (name == "source") $ warnMsg id "'source' in place of '.' is" + when (name == "source") $ warnMsg id 3046 "'source' in place of '.' is" when (name == "trap") $ let check token = sequence_ $ do @@ -356,12 +359,12 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do let upper = map toUpper str return $ do when (upper `elem` ["ERR", "DEBUG", "RETURN"]) $ - warnMsg (getId token) $ "trapping " ++ str ++ " is" + warnMsg (getId token) 3047 $ "trapping " ++ str ++ " is" when ("SIG" `isPrefixOf` upper) $ - warnMsg (getId token) + warnMsg (getId token) 3048 "prefixing signal names with 'SIG' is" when (not isDash && upper /= str) $ - warnMsg (getId token) + warnMsg (getId token) 3049 "using lower/mixed case for signal names is" in mapM_ check (drop 1 rest) @@ -370,13 +373,13 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do format <- rest !!! 0 -- flags are covered by allowedFlags let literal = onlyLiteralString format guard $ "%q" `isInfixOf` literal - return $ warnMsg (getId format) "printf %q is" + return $ warnMsg (getId format) 3050 "printf %q is" where unsupportedCommands = [ "let", "caller", "builtin", "complete", "compgen", "declare", "dirs", "disown", "enable", "mapfile", "readarray", "pushd", "popd", "shopt", "suspend", "typeset" - ] ++ if not isDash then ["local"] else [] + ] allowedFlags = Map.fromList [ ("cd", Just ["L", "P"]), ("exec", Just []), @@ -394,9 +397,9 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do ("wait", Just []) ] bashism t@(T_SourceCommand id src _) - | getCommandName src == Just "source" = warnMsg id "'source' in place of '.' is" + | getCommandName src == Just "source" = warnMsg id 3051 "'source' in place of '.' is" bashism (TA_Expansion _ (T_Literal id str : _)) - | str `matches` radix = warnMsg id "arithmetic base conversion is" + | str `matches` radix = warnMsg id 3052 "arithmetic base conversion is" where radix = mkRegex "^[0-9]+#" bashism _ = return () From c4cc2debb78b4448d37a672735f0c20a76cd7d51 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 7 Sep 2020 21:05:49 -0700 Subject: [PATCH 086/459] Improve compatibility checks --- src/ShellCheck/Checks/ShellSupport.hs | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 8b8e095..c4db9ce 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -180,6 +180,7 @@ prop_checkBashisms95 = verify checkBashisms "#!/bin/sh\necho $_" prop_checkBashisms96 = verifyNot checkBashisms "#!/bin/dash\necho $_" prop_checkBashisms97 = verify checkBashisms "#!/bin/sh\necho ${var,}" prop_checkBashisms98 = verify checkBashisms "#!/bin/sh\necho ${var^^}" +prop_checkBashisms99 = verifyNot checkBashisms "#!/bin/dash\necho [^f]oo" checkBashisms = ForShell [Sh, Dash] $ \t -> do params <- ask kludge params t @@ -234,11 +235,11 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do where file = onlyLiteralString word isNetworked = any (`isPrefixOf` file) ["/dev/tcp", "/dev/udp"] - bashism (T_Glob id str) | "[^" `isInfixOf` str = + bashism (T_Glob id str) | not isDash && "[^" `isInfixOf` str = warnMsg id 3026 "^ in place of ! in glob bracket expressions is" bashism t@(TA_Variable id str _) | isBashVariable str = - warnMsg id 3027 $ str ++ " is" + warnMsg id 3028 $ str ++ " is" bashism t@(T_DollarBraced id _ token) = do mapM_ check expansion @@ -247,8 +248,8 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do where str = concat $ oversimplify token var = getBracedReference str - check (regex, feature) = - when (isJust $ matchRegex regex str) $ warnMsg id 3053 feature + check (regex, code, feature) = + when (isJust $ matchRegex regex str) $ warnMsg id code feature bashism t@(T_Pipe id "|&") = warnMsg id 3029 "|& in place of 2>&1 | is" @@ -406,14 +407,14 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do varChars="_0-9a-zA-Z" expansion = let re = mkRegex in [ - (re $ "^![" ++ varChars ++ "]", "indirect expansion is"), - (re $ "^[" ++ varChars ++ "]+\\[.*\\]$", "array references are"), - (re $ "^![" ++ varChars ++ "]+\\[[*@]]$", "array key expansion is"), - (re $ "^![" ++ varChars ++ "]+[*@]$", "name matching prefixes are"), - (re $ "^[" ++ varChars ++ "*@]+:[^-=?+]", "string indexing is"), - (re $ "^([*@][%#]|#[@*])", "string operations on $@/$* are"), - (re $ "^[" ++ varChars ++ "*@]+(\\[.*\\])?[,^]", "case modification is"), - (re $ "^[" ++ varChars ++ "*@]+(\\[.*\\])?/", "string replacement is") + (re $ "^![" ++ varChars ++ "]", 3053, "indirect expansion is"), + (re $ "^[" ++ varChars ++ "]+\\[.*\\]$", 3054, "array references are"), + (re $ "^![" ++ varChars ++ "]+\\[[*@]]$", 3055, "array key expansion is"), + (re $ "^![" ++ varChars ++ "]+[*@]$", 3056, "name matching prefixes are"), + (re $ "^[" ++ varChars ++ "*@]+:[^-=?+]", 3057, "string indexing is"), + (re $ "^([*@][%#]|#[@*])", 3058, "string operations on $@/$* are"), + (re $ "^[" ++ varChars ++ "*@]+(\\[.*\\])?[,^]", 3059, "case modification is"), + (re $ "^[" ++ varChars ++ "*@]+(\\[.*\\])?/", 3060, "string replacement is") ] bashVars = [ "OSTYPE", "MACHTYPE", "HOSTTYPE", "HOSTNAME", From 218deb6d01ec327fdf91d8d3a00f1ecccfc9df44 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 8 Sep 2020 19:30:13 -0700 Subject: [PATCH 087/459] Update SC2091/SC2092 message and ignore in quotes. --- src/ShellCheck/Analytics.hs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index e0d33bb..c39f8c8 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1692,19 +1692,17 @@ checkSpuriousExec _ = doLists prop_checkSpuriousExpansion1 = verify checkSpuriousExpansion "if $(true); then true; fi" -prop_checkSpuriousExpansion2 = verify checkSpuriousExpansion "while \"$(cmd)\"; do :; done" prop_checkSpuriousExpansion3 = verifyNot checkSpuriousExpansion "$(cmd) --flag1 --flag2" prop_checkSpuriousExpansion4 = verify checkSpuriousExpansion "$((i++))" checkSpuriousExpansion _ (T_SimpleCommand _ _ [T_NormalWord _ [word]]) = check word where check word = case word of T_DollarExpansion id _ -> - warn id 2091 "Remove surrounding $() to avoid executing output." + warn id 2091 "Remove surrounding $() to avoid executing output (or use eval if intentional)." T_Backticked id _ -> - warn id 2092 "Remove backticks to avoid executing output." + warn id 2092 "Remove backticks to avoid executing output (or use eval if intentional)." T_DollarArithmetic id _ -> err id 2084 "Remove '$' or use '_=$((expr))' to avoid executing output." - T_DoubleQuoted id [subword] -> check subword _ -> return () checkSpuriousExpansion _ _ = return () From 8d999265541e28a1fa6f73364d70cc4cbcc70559 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 18 Oct 2020 15:15:31 -0700 Subject: [PATCH 088/459] Recognize `local -x` similarly to `export` (fixes #2069) --- src/ShellCheck/Analytics.hs | 1 + src/ShellCheck/AnalyzerLib.hs | 3 +++ 2 files changed, 4 insertions(+) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index c39f8c8..e5287f8 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2134,6 +2134,7 @@ prop_checkUnused13= verifyNotTree checkUnusedAssignments "x=(1); (( x[0] ))" prop_checkUnused14= verifyNotTree checkUnusedAssignments "x=(1); n=0; echo ${x[n]}" prop_checkUnused15= verifyNotTree checkUnusedAssignments "x=(1); n=0; (( x[n] ))" prop_checkUnused16= verifyNotTree checkUnusedAssignments "foo=5; declare -x foo" +prop_checkUnused16b= verifyNotTree checkUnusedAssignments "f() { local -x foo; foo=42; bar; }; f" prop_checkUnused17= verifyNotTree checkUnusedAssignments "read -i 'foo' -e -p 'Input: ' bar; $bar;" prop_checkUnused18= verifyNotTree checkUnusedAssignments "a=1; arr=( [$a]=42 ); echo \"${arr[@]}\"" prop_checkUnused19= verifyNotTree checkUnusedAssignments "a=1; let b=a+1; echo $b" diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index c9954d1..724857c 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -547,6 +547,9 @@ getReferencedVariableCommand base@(T_SimpleCommand _ _ (T_NormalWord _ (T_Litera (not $ any (`elem` flags) ["f", "F"]) then concatMap getReference rest else [] + "local" -> if "x" `elem` flags + then concatMap getReference rest + else [] "trap" -> case rest of head:_ -> map (\x -> (base, head, x)) $ getVariablesFromLiteralToken head From f100c2939e7fd50975b20d3c7e7cd9e635b1b428 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 18 Oct 2020 20:36:48 -0700 Subject: [PATCH 089/459] Rewrite getopts style option parser --- src/ShellCheck/ASTLib.hs | 103 ++++++++++++++++++++++++------ src/ShellCheck/Analytics.hs | 6 +- src/ShellCheck/AnalyzerLib.hs | 4 +- src/ShellCheck/Checks/Commands.hs | 39 +++++++++-- 4 files changed, 120 insertions(+), 32 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 0052ef5..a09bf38 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -30,6 +30,8 @@ import Data.List import Data.Maybe import qualified Data.Map as Map +arguments (T_SimpleCommand _ _ (cmd:args)) = args + -- Is this a type of loop? isLoop t = case t of T_WhileExpression {} -> True @@ -135,32 +137,91 @@ isUnquotedFlag token = fromMaybe False $ do str <- getLeadingUnquotedString token return $ "-" `isPrefixOf` str --- getGnuOpts "erd:u:" will parse a SimpleCommand like --- read -re -d : -u 3 bar +-- getGnuOpts "erd:u:" will parse a list of arguments tokens like `read` +-- -re -d : -u 3 bar -- into --- Just [("r", -re), ("e", -re), ("d", :), ("u", 3), ("", bar)] --- where flags with arguments map to arguments, while others map to themselves. --- Any unrecognized flag will result in Nothing. -getGnuOpts str t = getOpts str $ getAllFlags t -getBsdOpts str t = getOpts str $ getLeadingFlags t -getOpts :: String -> [(Token, String)] -> Maybe [(String, Token)] -getOpts string flags = process flags +-- Just [("r", (-re, -re)), ("e", (-re, -re)), ("d", (-d,:)), ("u", (-u,3)), ("", (bar,bar))] +-- +-- Each string flag maps to a tuple of (flag, argument), where argument=flag if it +-- doesn't take a specific one. +-- +-- Any unrecognized flag will result in Nothing. The exception is if arbitraryLongOpts +-- is set, in which case --anything will map to "anything". +getGnuOpts :: String -> [Token] -> Maybe [(String, (Token, Token))] +getGnuOpts str args = getOpts (True, False) str [] args + +-- As above, except the first non-arg string will treat the rest as arguments +getBsdOpts :: String -> [Token] -> Maybe [(String, (Token, Token))] +getBsdOpts str args = getOpts (False, False) str [] args + +-- Tests for this are in Commands.hs where it's more frequently used +getOpts :: + -- Behavioral config: gnu style, allow arbitrary long options + (Bool, Bool) + -- A getopts style string + -> String + -- List of long options and whether they take arguments + -> [(String, Bool)] + -- List of arguments (excluding command) + -> [Token] + -- List of flags to tuple of (optionToken, valueToken) + -> Maybe [(String, (Token, Token))] + +getOpts (gnu, arbitraryLongOpts) string longopts args = process args where flagList (c:':':rest) = ([c], True) : flagList rest flagList (c:rest) = ([c], False) : flagList rest - flagList [] = [] + flagList [] = longopts flagMap = Map.fromList $ ("", False) : flagList string process [] = return [] - process ((token1, flag):rest1) = do - takesArg <- Map.lookup flag flagMap - (token, rest) <- if takesArg - then case rest1 of - (token2, ""):rest2 -> return (token2, rest2) - _ -> fail "takesArg without valid arg" - else return (token1, rest1) - more <- process rest - return $ (flag, token) : more + process (token:rest) = do + case getLiteralStringDef "\0" token of + '-':'-':[] -> return $ listToArgs rest + '-':'-':word -> do + let (name, arg) = span (/= '=') word + needsArg <- + if arbitraryLongOpts + then return $ Map.findWithDefault False name flagMap + else Map.lookup name flagMap + + if needsArg && null arg + then + case rest of + (arg:rest2) -> do + more <- process rest2 + return $ (name, (token, arg)) : more + _ -> fail "Missing arg" + else do + more <- process rest + -- Consider splitting up token to get arg + return $ (name, (token, token)) : more + '-':opts -> shortToOpts opts token rest + arg -> + if gnu + then do + more <- process rest + return $ ("", (token, token)):more + else return $ listToArgs (token:rest) + + shortToOpts opts token args = + case opts of + c:rest -> do + needsArg <- Map.lookup [c] flagMap + case () of + _ | needsArg && null rest -> do + (next:restArgs) <- return args + more <- process restArgs + return $ ([c], (token, next)):more + _ | needsArg -> do + more <- process args + return $ ([c], (token, token)):more + _ -> do + more <- shortToOpts rest token args + return $ ([c], (token, token)):more + [] -> process args + + listToArgs = map (\x -> ("", (x, x))) -- Is this an expansion of multiple items of an array? isArrayExpansion (T_DollarBraced _ _ l) = @@ -362,8 +423,8 @@ getCommandNameAndToken direct t = fromMaybe (Nothing, t) $ do "builtin" -> firstArg "command" -> firstArg "exec" -> do - opts <- getBsdOpts "cla:" cmd - (_, t) <- listToMaybe $ filter (null . fst) opts + opts <- getBsdOpts "cla:" args + (_, (t, _)) <- listToMaybe $ filter (null . fst) opts return t _ -> fail "" diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index e5287f8..cf7bcb1 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2925,8 +2925,8 @@ checkReadWithoutR _ t@T_SimpleCommand {} | t `isUnqualifiedCommand` "read" where flags = getAllFlags t has_t0 = Just "0" == do - parsed <- getOpts flagsForRead flags - t <- lookup "t" parsed + parsed <- getGnuOpts flagsForRead $ arguments t + (_, t) <- lookup "t" parsed getLiteralString t checkReadWithoutR _ _ = return () @@ -3383,7 +3383,7 @@ checkPipeToNowhere params t = commandSpecificException name cmd = case name of - "du" -> any (`elem` ["exclude-from", "files0-from"]) $ lt $ map snd $ getAllFlags cmd + "du" -> any (`elem` ["exclude-from", "files0-from"]) $ map snd $ getAllFlags cmd _ -> False warnAboutDupes (n, list@(_:_:_)) = diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 724857c..0f40b3b 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -678,13 +678,13 @@ getModifiedVariableCommand base@(T_SimpleCommand id cmdPrefix (T_NormalWord _ (T where parseArgs :: Maybe (Token, Token, String, DataType) parseArgs = do - args <- getGnuOpts "d:n:O:s:u:C:c:t" base + args <- getGnuOpts "d:n:O:s:u:C:c:t" rest let names = map snd $ filter (\(x,y) -> null x) args if null names then return (base, base, "MAPFILE", DataArray SourceExternal) else do - first <- listToMaybe names + (_, first) <- listToMaybe names name <- getLiteralString first guard $ isVariableName name return (base, first, name, DataArray SourceExternal) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index b4e8b3e..9af7748 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -53,8 +53,6 @@ verify :: CommandCheck -> String -> Bool verify f s = producesComments (getChecker [f]) s == Just True verifyNot f s = producesComments (getChecker [f]) s == Just False -arguments (T_SimpleCommand _ _ (cmd:args)) = args - commandChecks :: [CommandCheck] commandChecks = [ checkTr @@ -116,6 +114,35 @@ prop_verifyOptionalExamples = all check optionalCommandChecks verify check (cdPositive desc) && verifyNot check (cdNegative desc) +-- Run a check against the getopt parser. If it fails, the lists are empty. +checkGetOpts str flags args f = + flags == actualFlags && args == actualArgs + where + toTokens = map (T_Literal (Id 0)) . words + opts = fromMaybe [] $ f (toTokens str) + actualFlags = filter (not . null) $ map fst opts + actualArgs = map (\(_, (_, x)) -> onlyLiteralString x) $ filter (null . fst) opts + +-- Short options +prop_checkGetOptsS1 = checkGetOpts "-f x" ["f"] [] $ getOpts (True, True) "f:" [] +prop_checkGetOptsS2 = checkGetOpts "-fx" ["f"] [] $ getOpts (True, True) "f:" [] +prop_checkGetOptsS3 = checkGetOpts "-f -x" ["f", "x"] [] $ getOpts (True, True) "fx" [] +prop_checkGetOptsS4 = checkGetOpts "-f -x" ["f"] [] $ getOpts (True, True) "f:" [] +prop_checkGetOptsS5 = checkGetOpts "-fx" [] [] $ getOpts (True, True) "fx:" [] + +-- Long options +prop_checkGetOptsL1 = checkGetOpts "--foo=bar baz" ["foo"] ["baz"] $ getOpts (True, False) "" [("foo", True)] +prop_checkGetOptsL2 = checkGetOpts "--foo bar baz" ["foo"] ["baz"] $ getOpts (True, False) "" [("foo", True)] +prop_checkGetOptsL3 = checkGetOpts "--foo baz" ["foo"] ["baz"] $ getOpts (True, True) "" [] +prop_checkGetOptsL4 = checkGetOpts "--foo baz" [] [] $ getOpts (True, False) "" [] + +-- Know when to terminate +prop_checkGetOptsT1 = checkGetOpts "-a x -b" ["a", "b"] ["x"] $ getOpts (True, True) "ab" [] +prop_checkGetOptsT2 = checkGetOpts "-a x -b" ["a"] ["x","-b"] $ getOpts (False, True) "ab" [] +prop_checkGetOptsT3 = checkGetOpts "-a -- -b" ["a"] ["-b"] $ getOpts (True, True) "ab" [] +prop_checkGetOptsT4 = checkGetOpts "-a -- -b" ["a", "b"] [] $ getOpts (True, True) "a:b" [] + + buildCommandMap :: [CommandCheck] -> Map.Map CommandName (Token -> Analysis) buildCommandMap = foldl' addCheck Map.empty where @@ -694,8 +721,8 @@ checkReadExpansions = CommandCheck (Exactly "read") check where options = getGnuOpts flagsForRead getVars cmd = fromMaybe [] $ do - opts <- options cmd - return [y | (x,y) <- opts, null x || x == "a"] + opts <- options $ arguments cmd + return [y | (x,(_, y)) <- opts, null x || x == "a"] check cmd = mapM_ warning $ getVars cmd warning t = sequence_ $ do @@ -1070,8 +1097,8 @@ prop_checkSudoArgs7 = verifyNot checkSudoArgs "sudo docker export foo" checkSudoArgs = CommandCheck (Basename "sudo") f where f t = sequence_ $ do - opts <- parseOpts t - let nonFlags = [x | ("",x) <- opts] + opts <- parseOpts $ arguments t + let nonFlags = [x | ("",(x, _)) <- opts] commandArg <- nonFlags !!! 0 command <- getLiteralString commandArg guard $ command `elem` builtins From 3104cec770a98d057fab3827f584c04486afbbb3 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 18 Oct 2020 22:08:40 -0700 Subject: [PATCH 090/459] SC2267: Warn about xargs -i (fixes #2058) --- CHANGELOG.md | 1 + src/ShellCheck/Checks/Commands.hs | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d4b9145..446303b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - SC2262/SC2263: Warn about aliases declared and used in the same parsing unit - SC2264: Warn about wrapper functions that blatantly recurse - SC2265/SC2266: Warn when using & or | with test statements +- SC2267: Warn when using xargs -i instead of -I ### Fixed - SC1072/SC1073 now respond to disable annotations, though ignoring parse errors diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 9af7748..aa4edd4 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -94,6 +94,7 @@ commandChecks = [ ,checkSudoArgs ,checkSourceArgs ,checkChmodDashr + ,checkXargsDashi ] optionalChecks = map fst optionalCommandChecks @@ -1129,5 +1130,18 @@ checkChmodDashr = CommandCheck (Basename "chmod") f guard $ flag == "-r" return $ warn (getId t) 2253 "Use -R to recurse, or explicitly a-r to remove read permissions." +prop_checkXargsDashi1 = verify checkXargsDashi "xargs -i{} echo {}" +prop_checkXargsDashi2 = verifyNot checkXargsDashi "xargs -I{} echo {}" +prop_checkXargsDashi3 = verifyNot checkXargsDashi "xargs sed -i -e foo" +prop_checkXargsDashi4 = verify checkXargsDashi "xargs -e sed -i foo" +prop_checkXargsDashi5 = verifyNot checkXargsDashi "xargs -x sed -i foo" +checkXargsDashi = CommandCheck (Basename "xargs") f + where + f t = sequence_ $ do + opts <- parseOpts $ arguments t + (option, value) <- lookup "i" opts + return $ info (getId option) 2267 "GNU xargs -i is deprecated in favor of -I{}" + parseOpts = getBsdOpts "0oprtxadR:S:J:L:l:n:P:s:e:E:i:I:" + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) From 256457c47af7c2b80f1ebb9cbd67fa8e47e789d5 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 18 Oct 2020 22:57:16 -0700 Subject: [PATCH 091/459] Use getopts parser to find 'read' arrays (fixes #2073) --- src/ShellCheck/Analytics.hs | 3 ++- src/ShellCheck/AnalyzerLib.hs | 22 ++++++++-------------- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index cf7bcb1..b5a8a16 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -851,7 +851,8 @@ prop_checkArrayWithoutIndex6 = verifyTree checkArrayWithoutIndex "echo $PIPESTAT prop_checkArrayWithoutIndex7 = verifyTree checkArrayWithoutIndex "a=(a b); a+=c" prop_checkArrayWithoutIndex8 = verifyTree checkArrayWithoutIndex "declare -a foo; foo=bar;" prop_checkArrayWithoutIndex9 = verifyTree checkArrayWithoutIndex "read -r -a arr <<< 'foo bar'; echo \"$arr\"" -prop_checkArrayWithoutIndex10= verifyTree checkArrayWithoutIndex "read -ra arr <<< 'foo bar'; echo \"$arr\"" +prop_checkArrayWithoutIndex10 = verifyTree checkArrayWithoutIndex "read -ra arr <<< 'foo bar'; echo \"$arr\"" +prop_checkArrayWithoutIndex11 = verifyNotTree checkArrayWithoutIndex "read -rpfoobar r; r=42" checkArrayWithoutIndex params _ = doVariableFlowAnalysis readF writeF defaultMap (variableFlow params) where diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 0f40b3b..dc081db 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -578,10 +578,14 @@ getModifiedVariableCommand base@(T_SimpleCommand id cmdPrefix (T_NormalWord _ (T "builtin" -> getModifiedVariableCommand $ T_SimpleCommand id cmdPrefix rest "read" -> - let params = map getLiteral rest - readArrayVars = getReadArrayVariables rest - in - catMaybes $ takeWhile isJust (reverse params) ++ readArrayVars + let fallback = catMaybes $ takeWhile isJust (reverse $ map getLiteral rest) + in fromMaybe fallback $ do + parsed <- getGnuOpts flagsForRead rest + case lookup "a" parsed of + Just (_, var) -> (:[]) <$> getLiteralArray var + Nothing -> return $ catMaybes $ + map (getLiteral . snd . snd) $ filter (null . fst) parsed + "getopts" -> case rest of opts:var:_ -> maybeToList $ getLiteral var @@ -698,16 +702,6 @@ getModifiedVariableCommand base@(T_SimpleCommand id cmdPrefix (T_NormalWord _ (T guard $ isVariableName name return (name, arg) - -- get all the array variables used in read, e.g. read -a arr - getReadArrayVariables args = - map (getLiteralArray . snd) - (filter (isArrayFlag . fst) (zip args (tail args))) - - isArrayFlag x = case getLiteralString x of - Just ('-':'-':_) -> False - Just ('-':str) -> 'a' `elem` str - _ -> False - -- get the FLAGS_ variable created by a shflags DEFINE_ call getFlagVariable (n:v:_) = do name <- getLiteralString n From 28d3279ba6a86c8dfc3f9d3c6262f58f57bc09dd Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 19 Oct 2020 18:25:18 -0700 Subject: [PATCH 092/459] Optional style warning about [ x$var = xval ] --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 42 +++++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 446303b..1e0eff4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - SC2264: Warn about wrapper functions that blatantly recurse - SC2265/SC2266: Warn when using & or | with test statements - SC2267: Warn when using xargs -i instead of -I +- Optional avoid-x-comparisons: Style warning SC2268 for `[ x$var = xval ]` ### Fixed - SC1072/SC1073 now respond to disable annotations, though ignoring parse errors diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index b5a8a16..f55b704 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -240,6 +240,13 @@ optionalTreeChecks = [ cdPositive = "echo $VAR", cdNegative = "VAR=hello; echo $VAR" }, checkUnassignedReferences' True) + + ,(newCheckDescription { + cdName = "avoid-x-comparisons", + cdDescription = "Warn about 'x'-prefix in comparisons", + cdPositive = "[ \"x$var\" = xval ]", + cdNegative = "[ \"$var\" = val ]" + }, nodeChecksToTreeCheck [checkComparisonWithLeadingX]) ] optionalCheckMap :: Map.Map String (Parameters -> Token -> [TokenComment]) @@ -3926,6 +3933,41 @@ checkBadTestAndOr params t = T_Annotation _ _ t -> isTest t _ -> False +prop_checkComparisonWithLeadingX1 = verify checkComparisonWithLeadingX "[ x$foo = xlol ]" +prop_checkComparisonWithLeadingX2 = verify checkComparisonWithLeadingX "test x$foo = xlol" +prop_checkComparisonWithLeadingX3 = verifyNot checkComparisonWithLeadingX "[ $foo = xbar ]" +prop_checkComparisonWithLeadingX4 = verifyNot checkComparisonWithLeadingX "test $foo = xbar" +prop_checkComparisonWithLeadingX5 = verify checkComparisonWithLeadingX "[ \"x$foo\" = 'xlol' ]" +prop_checkComparisonWithLeadingX6 = verify checkComparisonWithLeadingX "[ x\"$foo\" = x'lol' ]" +checkComparisonWithLeadingX params t = + case t of + TC_Binary id typ op lhs rhs | op == "=" || op == "==" -> + check lhs rhs + T_SimpleCommand _ _ [cmd, lhs, op, rhs] | + getLiteralString cmd == Just "test" && + getLiteralString op `elem` [Just "=", Just "=="] -> + check lhs rhs + _ -> return () + where + msg = "Avoid outdated x-prefix in comparisons as it no longer serves a purpose." + check lhs rhs = sequence_ $ do + l <- fixLeadingX lhs + r <- fixLeadingX rhs + return $ styleWithFix (getId lhs) 2268 msg $ fixWith [l, r] + + fixLeadingX token = + case getWordParts token of + T_Literal id ('x':_):_ -> + case token of + -- The side is a single, unquoted x, so we have to quote + T_NormalWord _ [T_Literal id "x"] -> + return $ replaceStart id params 1 "\"\"" + -- Otherwise we can just delete it + _ -> return $ replaceStart id params 1 "" + T_SingleQuoted id ('x':_):_ -> + -- Replace the single quote and x + return $ replaceStart id params 2 "'" + _ -> Nothing return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) From 619662adb63e027866a7511e695b87d8ca069654 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Fri, 23 Oct 2020 17:29:04 -0700 Subject: [PATCH 093/459] Add readonly to SC2155 This adds a warning for readonly masking the return value of function. This is mentioned in the wiki https://github.com/koalaman/shellcheck/wiki/SC2155#problematic-code-in-the-case-of-readonly but didn't actually produce a warning. Fixes https://github.com/koalaman/shellcheck/issues/1336 --- src/ShellCheck/Analytics.hs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index f55b704..c894075 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2902,11 +2902,14 @@ checkTestArgumentSplitting params t = prop_checkMaskedReturns1 = verify checkMaskedReturns "f() { local a=$(false); }" prop_checkMaskedReturns2 = verify checkMaskedReturns "declare a=$(false)" prop_checkMaskedReturns3 = verify checkMaskedReturns "declare a=\"`false`\"" -prop_checkMaskedReturns4 = verifyNot checkMaskedReturns "declare a; a=$(false)" -prop_checkMaskedReturns5 = verifyNot checkMaskedReturns "f() { local -r a=$(false); }" +prop_checkMaskedReturns4 = verify checkMaskedReturns "readonly a=$(false)" +prop_checkMaskedReturns5 = verify checkMaskedReturns "readonly a=\"`false`\"" +prop_checkMaskedReturns6 = verifyNot checkMaskedReturns "declare a; a=$(false)" +prop_checkMaskedReturns7 = verifyNot checkMaskedReturns "f() { local -r a=$(false); }" +prop_checkMaskedReturns8 = verifyNot checkMaskedReturns "a=$(false); readonly a" checkMaskedReturns _ t@(T_SimpleCommand id _ (cmd:rest)) = sequence_ $ do name <- getCommandName t - guard $ name `elem` ["declare", "export"] + guard $ name `elem` ["declare", "export", "readonly"] || name == "local" && "r" `notElem` map snd (getAllFlags t) return $ mapM_ checkArgs rest where From 4b0e5ca11933fb7735977c032ec80775ab7640aa Mon Sep 17 00:00:00 2001 From: Artur Klauser Date: Sun, 5 Apr 2020 21:11:35 +0200 Subject: [PATCH 094/459] Simplify .prepare-deploy Reduce amount of duplicated code. --- .prepare_deploy | 31 +++++-------------------------- 1 file changed, 5 insertions(+), 26 deletions(-) diff --git a/.prepare_deploy b/.prepare_deploy index 226c16d..a2ce361 100755 --- a/.prepare_deploy +++ b/.prepare_deploy @@ -27,35 +27,14 @@ do zip "${file%.*}.zip" README.txt LICENSE.txt "$file" done -for file in *.linux-x86_64 +for file in *.{linux,darwin}-* do base="${file%.*}" + ext="${file##*.}" + os="${ext%-*}" + arch="${ext##*-}" cp "$file" "shellcheck" - tar -cJf "$base.linux.x86_64.tar.xz" --transform="s:^:$base/:" README.txt LICENSE.txt shellcheck - rm "shellcheck" -done - -for file in *.linux-aarch64 -do - base="${file%.*}" - cp "$file" "shellcheck" - tar -cJf "$base.linux.aarch64.tar.xz" --transform="s:^:$base/:" README.txt LICENSE.txt shellcheck - rm "shellcheck" -done - -for file in *.linux-armv6hf -do - base="${file%.*}" - cp "$file" "shellcheck" - tar -cJf "$base.linux.armv6hf.tar.xz" --transform="s:^:$base/:" README.txt LICENSE.txt shellcheck - rm "shellcheck" -done - -for file in *.darwin-x86_64 -do - base="${file%.*}" - cp "$file" "shellcheck" - tar -cJf "$base.darwin.x86_64.tar.xz" --transform="s:^:$base/:" README.txt LICENSE.txt shellcheck + tar -cJf "$base.$os.$arch.tar.xz" --transform="s:^:$base/:" README.txt LICENSE.txt shellcheck rm "shellcheck" done From 65044c25681038dc061ee125d161b1325abbec10 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 28 Oct 2020 13:36:10 -0700 Subject: [PATCH 095/459] SC2095: Also warn if the command is backgrounded --- src/ShellCheck/Analytics.hs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index c894075..ed626d9 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2361,6 +2361,7 @@ prop_checkWhileReadPitfalls11 = verifyNot checkWhileReadPitfalls "while read foo prop_checkWhileReadPitfalls12 = verifyNot checkWhileReadPitfalls "while read foo\ndo\nmplayer foo.ogv << EOF\nq\nEOF\ndone" prop_checkWhileReadPitfalls13 = verify checkWhileReadPitfalls "while read foo; do x=$(ssh host cmd); done" prop_checkWhileReadPitfalls14 = verify checkWhileReadPitfalls "while read foo; do echo $(ssh host cmd) < /dev/null; done" +prop_checkWhileReadPitfalls15 = verify checkWhileReadPitfalls "while read foo; do ssh $foo cmd & done" checkWhileReadPitfalls params (T_WhileExpression id [command] contents) | isStdinReadCommand command = @@ -2411,6 +2412,7 @@ checkWhileReadPitfalls params (T_WhileExpression id [command] contents) warnWithFix (getId cmd) 2095 ("Use " ++ name ++ " " ++ flag ++ " to prevent " ++ name ++ " from swallowing stdin.") (fix flag cmd) + checkMuncher (T_Backgrounded _ t) = checkMuncher t checkMuncher _ = return () stdinRedirect (T_FdRedirect _ fd op) From 18e80284ec4f186e9cb061fb3a5210ed9901fb4c Mon Sep 17 00:00:00 2001 From: George Plymale II Date: Tue, 1 Dec 2020 16:15:22 -0500 Subject: [PATCH 096/459] add macports as installation option in README.md --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 05ce5ad..5459c53 100644 --- a/README.md +++ b/README.md @@ -172,6 +172,10 @@ On macOS (OS X) with Homebrew: brew install shellcheck +Or with MacPorts: + + sudo port install shellcheck + On OpenBSD: pkg_add shellcheck From b625562d609db2d5e10c0218445ada1f22e0b52f Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 2 Dec 2020 18:43:56 -0800 Subject: [PATCH 097/459] Add POSIX checks for more Bash-specific variables (fixes #2093) --- src/ShellCheck/Checks/ShellSupport.hs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index c4db9ce..1d373e5 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -417,9 +417,15 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do (re $ "^[" ++ varChars ++ "*@]+(\\[.*\\])?/", 3060, "string replacement is") ] bashVars = [ + -- This list deliberately excludes $BASH_VERSION as it's often used + -- for shell identification. "OSTYPE", "MACHTYPE", "HOSTTYPE", "HOSTNAME", "DIRSTACK", "EUID", "UID", "SHLVL", "PIPESTATUS", "SHELLOPTS", - "_" + "_", "BASHOPTS", "BASHPID", "BASH_ALIASES", "BASH_ARGC", + "BASH_ARGV", "BASH_ARGV0", "BASH_CMDS", "BASH_COMMAND", + "BASH_EXECUTION_STRING", "BASH_LINENO", "BASH_REMATCH", "BASH_SOURCE", + "BASH_SUBSHELL", "BASH_VERSINFO", "EPOCHREALTIME", "EPOCHSECONDS", + "FUNCNAME", "GROUPS", "MACHTYPE", "MAPFILE" ] bashDynamicVars = [ "RANDOM", "SECONDS" ] dashVars = [ "_" ] From 8e332ce87978e6f6afdfabcc15b382983ea32550 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 6 Dec 2020 20:30:43 -0800 Subject: [PATCH 098/459] Improve handling of trailing tokens for []/compounds (fixes #2091) --- CHANGELOG.md | 1 + src/ShellCheck/Parser.hs | 66 +++++++++++++++++++++++++++++----------- 2 files changed, 50 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e0eff4..fbedc20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ ### Fixed - SC1072/SC1073 now respond to disable annotations, though ignoring parse errors is still purely cosmetic and does not allow ShellCheck to continue. +- Improved error reporting for trailing tokens after ]/]] and compound commands ### Changed - Assignments are now parsed to spec, without leniency for leading $ or spaces diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 87efa25..dbadc7c 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -392,6 +392,8 @@ unexpecting s p = try $ notFollowedBy2 = unexpecting "" +isFollowedBy p = (lookAhead . try $ p *> return True) <|> return False + reluctantlyTill p end = (lookAhead (void (try end) <|> eof) >> return []) <|> do x <- p @@ -923,8 +925,9 @@ prop_readCondition20 = isOk readCondition "[[ echo_rc -eq 0 ]]" prop_readCondition21 = isOk readCondition "[[ $1 =~ ^(a\\ b)$ ]]" prop_readCondition22 = isOk readCondition "[[ $1 =~ \\.a\\.(\\.b\\.)\\.c\\. ]]" prop_readCondition23 = isOk readCondition "[[ -v arr[$var] ]]" -prop_readCondition24 = isWarning readCondition "[[ 1 == 2 ]]]" prop_readCondition25 = isOk readCondition "[[ lex.yy.c -ot program.l ]]" +prop_readCondition26 = isOk readScript "[[ foo ]]\\\n && bar" +prop_readCondition27 = not $ isOk readConditionCommand "[[ x ]] foo" readCondition = called "test expression" $ do opos <- getPosition start <- startSpan @@ -953,13 +956,7 @@ readCondition = called "test expression" $ do id <- endSpan start when (open == "[[" && close /= "]]") $ parseProblemAt cpos ErrorC 1033 "Test expression was opened with double [[ but closed with single ]. Make sure they match." when (open == "[" && close /= "]" ) $ parseProblemAt opos ErrorC 1034 "Test expression was opened with single [ but closed with double ]]. Make sure they match." - optional $ lookAhead $ do - pos <- getPosition - notFollowedBy2 readCmdWord <|> - parseProblemAt pos ErrorC 1136 - ("Unexpected characters after terminating " ++ close ++ ". Missing semicolon/linefeed?") spacing - many readCmdWord -- Read and throw away remainders to get then/do warnings. Fixme? return $ T_Condition id typ condition readAnnotationPrefix = do @@ -1617,6 +1614,7 @@ readArithmeticExpression = called "((..)) command" $ do c <- readArithmeticContents string "))" id <- endSpan start + spacing return (T_Arithmetic id c) -- If the next characters match prefix, try two different parsers and warn if the alternate parser had to be used @@ -1969,8 +1967,6 @@ readIoRedirect = do spacing return $ T_FdRedirect id n redir -readRedirectList = many1 readIoRedirect - prop_readHereString = isOk readHereString "<<< \"Hello $world\"" readHereString = called "here string" $ do start <- startSpan @@ -2300,6 +2296,7 @@ readPipe = do readCommand = choice [ readCompoundCommand, + readConditionCommand, readCoProc, readSimpleCommand ] @@ -2412,6 +2409,7 @@ readSubshell = called "explicit subshell" $ do allspacing char ')' <|> fail "Expected ) closing the subshell" id <- endSpan start + spacing return $ T_Subshell id list prop_readBraceGroup = isOk readBraceGroup "{ a; b | c | d; e; }" @@ -2432,6 +2430,7 @@ readBraceGroup = called "brace group" $ do parseProblem ErrorC 1056 "Expected a '}'. If you have one, try a ; or \\n in front of it." fail "Missing '}'" id <- endSpan start + spacing return $ T_BraceGroup id list prop_readBatsTest = isOk readBatsTest "@test 'can parse' {\n true\n}" @@ -2484,6 +2483,11 @@ readDoGroup kwId = do parseProblemAtId (getId doKw) ErrorC 1061 "Couldn't find 'done' for this 'do'." parseProblem ErrorC 1062 "Expected 'done' matching previously mentioned 'do'." return "Expected 'done'" + + optional . lookAhead $ do + pos <- getPosition + try $ string "<(" + parseProblemAt pos ErrorC 1142 "Use 'done < <(cmd)' to redirect from process substitution (currently missing one '<')." return commands @@ -2701,9 +2705,38 @@ readCoProc = called "coproc" $ do id <- endSpan start return $ T_CoProcBody id body - readPattern = (readPatternWord `thenSkip` spacing) `sepBy1` (char '|' `thenSkip` spacing) +prop_readConditionCommand = isOk readConditionCommand "[[ x ]] > foo 2>&1" +readConditionCommand = do + cmd <- readCondition + redirs <- many readIoRedirect + id <- getNextIdSpanningTokenList (cmd:redirs) + + pos <- getPosition + hasDashAo <- isFollowedBy $ do + c <- choice $ map (\s -> try $ string s) ["-o", "-a", "or", "and"] + posEnd <- getPosition + parseProblemAtWithEnd pos posEnd ErrorC 1139 $ + "Use " ++ alt c ++ " instead of '" ++ c ++ "' between test commands." + + -- If the next word is a keyword, readNormalWord will trigger a warning + hasKeyword <- isFollowedBy readKeyword + hasWord <- isFollowedBy readNormalWord + + when (hasWord && not (hasKeyword || hasDashAo)) $ do + -- We have other words following, and no error has been emitted. + posEnd <- getPosition + parseProblemAtWithEnd pos posEnd ErrorC 1140 "Unexpected parameters after condition. Missing &&/||, or bad expression?" + + return $ T_Redirecting id redirs cmd + where + alt "or" = "||" + alt "-o" = "||" + alt "and" = "&&" + alt "-a" = "&&" + alt _ = "|| or &&" + prop_readCompoundCommand = isOk readCompoundCommand "{ echo foo; }>/dev/null" readCompoundCommand = do cmd <- choice [ @@ -2711,7 +2744,6 @@ readCompoundCommand = do readAmbiguous "((" readArithmeticExpression readSubshell (\pos -> parseNoteAt pos ErrorC 1105 "Shells disambiguate (( differently or not at all. For subshell, add spaces around ( . For ((, fix parsing errors."), readSubshell, - readCondition, readWhileClause, readUntilClause, readIfClause, @@ -2721,15 +2753,15 @@ readCompoundCommand = do readBatsTest, readFunctionDefinition ] - spacing redirs <- many readIoRedirect id <- getNextIdSpanningTokenList (cmd:redirs) - unless (null redirs) $ optional $ do - lookAhead $ try (spacing >> needsSeparator) - parseProblem WarningC 1013 "Bash requires ; or \\n here, after redirecting nested compound commands." + optional . lookAhead $ do + notFollowedBy2 $ choice [readKeyword, g_Lbrace] + pos <- getPosition + many1 readNormalWord + posEnd <- getPosition + parseProblemAtWithEnd pos posEnd ErrorC 1141 "Unexpected tokens after compound command. Bad redirection or missing ;/&&/||/|?" return $ T_Redirecting id redirs cmd - where - needsSeparator = choice [ g_Then, g_Else, g_Elif, g_Fi, g_Do, g_Done, g_Esac, g_Rbrace ] readCompoundList = readTerm From 6ba1af089845a2c280a24e2dab58129391481175 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Fri, 11 Dec 2020 20:28:36 -0800 Subject: [PATCH 099/459] Warn when a variable is assigned to itself --- src/ShellCheck/Analytics.hs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index ed626d9..ef1ed2c 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -193,6 +193,7 @@ nodeChecks = [ ,checkModifiedArithmeticInRedirection ,checkBlatantRecursion ,checkBadTestAndOr + ,checkAssignToSelf ] optionalChecks = map fst optionalTreeChecks @@ -3974,5 +3975,26 @@ checkComparisonWithLeadingX params t = return $ replaceStart id params 2 "'" _ -> Nothing +prop_checkAssignToSelf1 = verify checkAssignToSelf "x=$x" +prop_checkAssignToSelf2 = verify checkAssignToSelf "x=${x}" +prop_checkAssignToSelf3 = verify checkAssignToSelf "x=\"$x\"" +prop_checkAssignToSelf4 = verifyNot checkAssignToSelf "x=$x mycmd" +checkAssignToSelf _ t = + case t of + T_SimpleCommand _ vars [] -> mapM_ check vars + _ -> return () + where + check t = + case t of + T_Assignment id Assign name [] t -> + case getWordParts t of + [T_DollarBraced _ _ b] -> do + when (Just name == getLiteralString b) $ + msg id + _ -> return () + _ -> return () + msg id = info id 2269 "This variable is assigned to itself, so the assignment does nothing." + + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) From cc3884cf9f81c7b709e89850167b59e03ac91266 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 12 Dec 2020 20:24:32 -0800 Subject: [PATCH 100/459] Support env -S/--split-string in shebangs (fixes #2105) --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 7 ++++++- src/ShellCheck/AnalyzerLib.hs | 8 ++++++-- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fbedc20..0617ab0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ - SC1072/SC1073 now respond to disable annotations, though ignoring parse errors is still purely cosmetic and does not allow ShellCheck to continue. - Improved error reporting for trailing tokens after ]/]] and compound commands +- `#!/usr/bin/env -S shell` is now handled correctly ### Changed - Assignments are now parsed to spec, without leniency for leading $ or spaces diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index ef1ed2c..2e1adca 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -546,9 +546,14 @@ indexOfSublists sub = f 0 prop_checkShebangParameters1 = verifyTree checkShebangParameters "#!/usr/bin/env bash -x\necho cow" prop_checkShebangParameters2 = verifyNotTree checkShebangParameters "#! /bin/sh -l " +prop_checkShebangParameters3 = verifyNotTree checkShebangParameters "#!/usr/bin/env -S bash -x\necho cow" +prop_checkShebangParameters4 = verifyNotTree checkShebangParameters "#!/usr/bin/env --split-string bash -x\necho cow" checkShebangParameters p (T_Annotation _ _ t) = checkShebangParameters p t checkShebangParameters _ (T_Script _ (T_Literal id sb) _) = - [makeComment ErrorC id 2096 "On most OS, shebangs can only specify a single parameter." | length (words sb) > 2] + [makeComment ErrorC id 2096 "On most OS, shebangs can only specify a single parameter." | isMultiWord] + where + isMultiWord = length (words sb) > 2 && not (sb `matches` re) + re = mkRegex "env +(-S|--split-string)" prop_checkShebang1 = verifyNotTree checkShebang "#!/usr/bin/env bash -x\necho cow" prop_checkShebang2 = verifyNotTree checkShebang "#! /bin/sh -l " diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index dc081db..38e9c1d 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -238,6 +238,8 @@ prop_determineShell5 = determineShellTest "#shellcheck shell=sh\nfoo" == Sh prop_determineShell6 = determineShellTest "#! /bin/sh" == Sh prop_determineShell7 = determineShellTest "#! /bin/ash" == Dash prop_determineShell8 = determineShellTest' (Just Ksh) "#!/bin/sh" == Sh +prop_determineShell9 = determineShellTest "#!/bin/env -S dash -x" == Dash +prop_determineShell10 = determineShellTest "#!/bin/env --split-string= dash -x" == Dash determineShellTest = determineShellTest' Nothing determineShellTest' fallbackShell = determineShell fallbackShell . fromJust . prRoot . pScript @@ -256,10 +258,12 @@ determineShell fallbackShell t = fromMaybe Bash $ executableFromShebang :: String -> String executableFromShebang = shellFor where - shellFor s | "/env " `isInfixOf` s = headOrDefault "" (drop 1 $ words s) + shellFor s | "/env " `isInfixOf` s = fromMaybe "" $ do + [flag, shell] <- matchRegex re s + return shell shellFor s | ' ' `elem` s = shellFor $ takeWhile (/= ' ') s shellFor s = reverse . takeWhile (/= '/') . reverse $ s - + re = mkRegex "/env +(-S|--split-string=?)? *([^ ]*)" -- Given a root node, make a map from Id to parent Token. From bd3299edd3b517f92f74c2e3327c9f6b72b31f7c Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 17 Dec 2020 20:30:39 -0800 Subject: [PATCH 101/459] Treat 'exec $1' like '$1' for the purpose of quoting (fixes #2068) --- src/ShellCheck/ASTLib.hs | 1 + src/ShellCheck/Analytics.hs | 2 ++ src/ShellCheck/AnalyzerLib.hs | 4 ++-- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index a09bf38..dcc9904 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -422,6 +422,7 @@ getCommandNameAndToken direct t = fromMaybe (Nothing, t) $ do "busybox" -> firstArg "builtin" -> firstArg "command" -> firstArg + "run" -> firstArg -- Used by bats "exec" -> do opts <- getBsdOpts "cla:" args (_, (t, _)) <- listToMaybe $ filter (null . fst) opts diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 2e1adca..2fb1253 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1853,6 +1853,8 @@ prop_checkSpacefulness37v = verifyTree checkVerboseSpacefulness "@test 'status' prop_checkSpacefulness38= verifyTree checkSpacefulness "a=; echo $a" prop_checkSpacefulness39= verifyNotTree checkSpacefulness "a=''\"\"''; b=x$a; echo $b" prop_checkSpacefulness40= verifyNotTree checkSpacefulness "a=$((x+1)); echo $a" +prop_checkSpacefulness41= verifyNotTree checkSpacefulness "exec $1 --flags" +prop_checkSpacefulness42= verifyNotTree checkSpacefulness "run $1 --flags" data SpaceStatus = SpaceSome | SpaceNone | SpaceEmpty deriving (Eq) instance Semigroup SpaceStatus where diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 38e9c1d..194bf18 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -370,8 +370,8 @@ usedAsCommandName tree token = go (getId token) (tail $ getPath tree token) | currentId == getId word = go id rest go currentId (T_DoubleQuoted id [word]:rest) | currentId == getId word = go id rest - go currentId (T_SimpleCommand _ _ (word:_):_) - | currentId == getId word = True + go currentId (t@(T_SimpleCommand _ _ (word:_)):_) = + getId word == currentId || getId (getCommandTokenOrThis t) == currentId go _ _ = False -- A list of the element and all its parents up to the root node. From 4e7e3f94565a5423aca0660ee822308d16a7f722 Mon Sep 17 00:00:00 2001 From: Pepe Iborra Date: Tue, 22 Dec 2020 09:15:57 +0000 Subject: [PATCH 102/459] Add Haddock markup to SystemInterface --- src/ShellCheck/Interface.hs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ShellCheck/Interface.hs b/src/ShellCheck/Interface.hs index 85d25c0..87346a1 100644 --- a/src/ShellCheck/Interface.hs +++ b/src/ShellCheck/Interface.hs @@ -73,15 +73,15 @@ import qualified Data.Map as Map data SystemInterface m = SystemInterface { - -- Read a file by filename, or return an error + -- | Read a file by filename, or return an error siReadFile :: String -> m (Either ErrorMessage String), - -- Given: + -- | Given: -- the current script, -- a list of source-path annotations in effect, -- and a sourced file, -- find the sourced file siFindSource :: String -> [String] -> String -> m FilePath, - -- Get the configuration file (name, contents) for a filename + -- | Get the configuration file (name, contents) for a filename siGetConfig :: String -> m (Maybe (FilePath, String)) } From 19355226e1d88abac37bb2eb5f366110ead0ecb1 Mon Sep 17 00:00:00 2001 From: Martin Bagge / brother Date: Sun, 27 Dec 2020 00:27:36 +0100 Subject: [PATCH 103/459] Change error 2076 to a warning. Implementing the suggestion by @pixarbuff #1985. --- src/ShellCheck/Analytics.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 2fb1253..a2ab577 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1193,8 +1193,8 @@ checkQuotedCondRegex _ (TC_Binary _ _ "=~" _ rhs) = where error t = unless (isConstantNonRe t) $ - err (getId t) 2076 - "Don't quote right-hand side of =~, it'll match literally rather than as a regex." + warn (getId t) 2076 + "Remove quotes from right-hand side of =~ to match as a regex rather than literally." re = mkRegex "[][*.+()|]" hasMetachars s = s `matches` re isConstantNonRe t = fromMaybe False $ do From 35033a9f2fdca88b916b62bbff284c218167d8f4 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Mon, 28 Dec 2020 16:22:53 -0500 Subject: [PATCH 104/459] Remove unnecessary use of Maybe from shellFor --- src/ShellCheck/AnalyzerLib.hs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 194bf18..9265a3f 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -258,9 +258,9 @@ determineShell fallbackShell t = fromMaybe Bash $ executableFromShebang :: String -> String executableFromShebang = shellFor where - shellFor s | "/env " `isInfixOf` s = fromMaybe "" $ do - [flag, shell] <- matchRegex re s - return shell + shellFor s | "/env " `isInfixOf` s = case matchRegex re s of + Just [flag, shell] -> shell + _ -> "" shellFor s | ' ' `elem` s = shellFor $ takeWhile (/= ' ') s shellFor s = reverse . takeWhile (/= '/') . reverse $ s re = mkRegex "/env +(-S|--split-string=?)? *([^ ]*)" From eaccd3d02c2e9c2f64d796801220ced38cce57fd Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Mon, 28 Dec 2020 16:32:10 -0500 Subject: [PATCH 105/459] Simplify parser --- src/ShellCheck/Parser.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index dbadc7c..104e0a4 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -392,7 +392,7 @@ unexpecting s p = try $ notFollowedBy2 = unexpecting "" -isFollowedBy p = (lookAhead . try $ p *> return True) <|> return False +isFollowedBy p = (lookAhead . try $ p $> True) <|> return False reluctantlyTill p end = (lookAhead (void (try end) <|> eof) >> return []) <|> do @@ -2715,7 +2715,7 @@ readConditionCommand = do pos <- getPosition hasDashAo <- isFollowedBy $ do - c <- choice $ map (\s -> try $ string s) ["-o", "-a", "or", "and"] + c <- choice $ try . string <$> ["-o", "-a", "or", "and"] posEnd <- getPosition parseProblemAtWithEnd pos posEnd ErrorC 1139 $ "Use " ++ alt c ++ " instead of '" ++ c ++ "' between test commands." From 46f177b5beb5c9862bfd5e3d10cc25a844af4c4c Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Mon, 28 Dec 2020 17:14:18 -0500 Subject: [PATCH 106/459] Simplify parseArgs --- src/ShellCheck/AnalyzerLib.hs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 9265a3f..a1e089a 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -687,12 +687,10 @@ getModifiedVariableCommand base@(T_SimpleCommand id cmdPrefix (T_NormalWord _ (T parseArgs :: Maybe (Token, Token, String, DataType) parseArgs = do args <- getGnuOpts "d:n:O:s:u:C:c:t" rest - let names = map snd $ filter (\(x,y) -> null x) args - if null names - then + case [y | ("",(_,y)) <- args] of + [] -> return (base, base, "MAPFILE", DataArray SourceExternal) - else do - (_, first) <- listToMaybe names + first:_ -> do name <- getLiteralString first guard $ isVariableName name return (base, first, name, DataArray SourceExternal) From 0607039d419a075df594fd5c38153e2d4863e834 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Mon, 28 Dec 2020 17:21:47 -0500 Subject: [PATCH 107/459] Simplify actualArgs --- src/ShellCheck/Checks/Commands.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index aa4edd4..f1996ef 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -122,7 +122,7 @@ checkGetOpts str flags args f = toTokens = map (T_Literal (Id 0)) . words opts = fromMaybe [] $ f (toTokens str) actualFlags = filter (not . null) $ map fst opts - actualArgs = map (\(_, (_, x)) -> onlyLiteralString x) $ filter (null . fst) opts + actualArgs = [onlyLiteralString x | ("", (_, x)) <- opts] -- Short options prop_checkGetOptsS1 = checkGetOpts "-f x" ["f"] [] $ getOpts (True, True) "f:" [] From cb4f4e7edc8721e12bd7a4ace9f88e8468961c53 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Mon, 28 Dec 2020 17:34:52 -0500 Subject: [PATCH 108/459] Use mapM_ instead of reimplementing it --- src/ShellCheck/Analytics.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 2fb1253..3d5692a 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -3808,7 +3808,7 @@ checkAliasUsedInSameParsingUnit params root = -- Group them by whether they start on the same line where the previous one ended units = groupByLink followsOnLine commands in - execWriter $ sequence_ $ map checkUnit units + execWriter $ mapM_ checkUnit units where lineSpan t = let m = tokenPositions params in do From 2c0766825e3e25589467c03435416f2fa3d9062f Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Mon, 28 Dec 2020 17:45:11 -0500 Subject: [PATCH 109/459] Implement groupByLink in terms of foldr --- src/ShellCheck/Analytics.hs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 3d5692a..70d7f8e 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -3858,13 +3858,13 @@ groupByLink :: (a -> a -> Bool) -> [a] -> [[a]] groupByLink f list = case list of [] -> [] - (x:xs) -> g x [] xs + (x:xs) -> foldr c n xs x [] where - g current span (next:rest) = + c next rest current span = if f current next - then g next (current:span) rest - else (reverse $ current:span) : g next [] rest - g current span [] = [reverse (current:span)] + then rest next (current:span) + else (reverse $ current:span) : rest next [] + n current span = [reverse (current:span)] prop_checkBlatantRecursion1 = verify checkBlatantRecursion ":(){ :|:& };:" From dfbcc9595e0b80e25c61bfedb7ad0347ff045395 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Mon, 28 Dec 2020 17:48:58 -0500 Subject: [PATCH 110/459] Use mapM instead of reimplementing it --- src/ShellCheck/Analytics.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 70d7f8e..945397c 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -3371,7 +3371,7 @@ checkPipeToNowhere params t = sequence_ $ do T_Redirecting _ redirs cmd <- return stage - fds <- sequence $ map getRedirectionFds redirs + fds <- mapM getRedirectionFds redirs let fdAndToken :: [(Integer, Token)] fdAndToken = From 848056367267e75cb3db0e45ec282a58105c0ed0 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Mon, 28 Dec 2020 17:55:54 -0500 Subject: [PATCH 111/459] Use syntactic sugar instead of building lists by hand --- src/ShellCheck/ASTLib.hs | 2 +- src/ShellCheck/Analytics.hs | 6 +++--- src/ShellCheck/Checks/Commands.hs | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index dcc9904..6e48bf3 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -177,7 +177,7 @@ getOpts (gnu, arbitraryLongOpts) string longopts args = process args process [] = return [] process (token:rest) = do case getLiteralStringDef "\0" token of - '-':'-':[] -> return $ listToArgs rest + "--" -> return $ listToArgs rest '-':'-':word -> do let (name, arg) = span (/= '=') word needsArg <- diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 945397c..c8473eb 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -394,7 +394,7 @@ prop_checkAssignAteCommand4 = verifyNot checkAssignAteCommand "A=foo ls -l" prop_checkAssignAteCommand5 = verify checkAssignAteCommand "PAGER=cat grep bar" prop_checkAssignAteCommand6 = verifyNot checkAssignAteCommand "PAGER=\"cat\" grep bar" prop_checkAssignAteCommand7 = verify checkAssignAteCommand "here=pwd" -checkAssignAteCommand _ (T_SimpleCommand id (T_Assignment _ _ _ _ assignmentTerm:[]) list) = +checkAssignAteCommand _ (T_SimpleCommand id [T_Assignment _ _ _ _ assignmentTerm] list) = -- Check if first word is intended as an argument (flag or glob). if firstWordIsArg list then @@ -426,7 +426,7 @@ checkArithmeticOpCommand _ _ = return () prop_checkWrongArit = verify checkWrongArithmeticAssignment "i=i+1" prop_checkWrongArit2 = verify checkWrongArithmeticAssignment "n=2; i=n*2" -checkWrongArithmeticAssignment params (T_SimpleCommand id (T_Assignment _ _ _ _ val:[]) []) = +checkWrongArithmeticAssignment params (T_SimpleCommand id [T_Assignment _ _ _ _ val] []) = sequence_ $ do str <- getNormalString val match <- matchRegex regex str @@ -2844,7 +2844,7 @@ checkTestArgumentSplitting params t = then -- Ksh appears to stop processing after unrecognized tokens, so operators -- will effectively work with globs, but only the first match. - when (op `elem` ['-':c:[] | c <- "bcdfgkprsuwxLhNOGRS" ]) $ + when (op `elem` [['-', c] | c <- "bcdfgkprsuwxLhNOGRS" ]) $ warn (getId token) 2245 $ op ++ " only applies to the first expansion of this glob. Use a loop to check any/all." else diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index f1996ef..771083e 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -916,7 +916,7 @@ checkWhileGetoptsCase = CommandCheck (Exactly "getopts") f fromGlob t = case t of - T_Glob _ ('[':c:']':[]) -> return [c] + T_Glob _ ['[', c, ']'] -> return [c] T_Glob _ "*" -> return "*" T_Glob _ "?" -> return "?" _ -> Nothing From e7820479f0502ea4571ec797d286c9037a81d967 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Mon, 28 Dec 2020 17:56:25 -0500 Subject: [PATCH 112/459] Use find --- src/ShellCheck/ASTLib.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 6e48bf3..c038ebe 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -425,7 +425,7 @@ getCommandNameAndToken direct t = fromMaybe (Nothing, t) $ do "run" -> firstArg -- Used by bats "exec" -> do opts <- getBsdOpts "cla:" args - (_, (t, _)) <- listToMaybe $ filter (null . fst) opts + (_, (t, _)) <- find (null . fst) opts return t _ -> fail "" From 34939ca0b7dce27472543466eeab1f68df317274 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Mon, 28 Dec 2020 18:00:14 -0500 Subject: [PATCH 113/459] Fuse map into any --- src/ShellCheck/Analytics.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index c8473eb..98fd5d7 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2481,7 +2481,7 @@ checkCharRangeGlob p t@(T_Glob id str) | where isCharClass str = "[" `isPrefixOf` str && "]" `isSuffixOf` str contents = dropNegation . drop 1 . take (length str - 1) $ str - hasDupes = any (>1) . map length . group . sort . filter (/= '-') $ contents + hasDupes = any ((>1) . length) . group . sort . filter (/= '-') $ contents dropNegation s = case s of '!':rest -> rest @@ -3404,7 +3404,7 @@ checkPipeToNowhere params t = commandSpecificException name cmd = case name of - "du" -> any (`elem` ["exclude-from", "files0-from"]) $ map snd $ getAllFlags cmd + "du" -> any ((`elem` ["exclude-from", "files0-from"]) . snd) $ getAllFlags cmd _ -> False warnAboutDupes (n, list@(_:_:_)) = From 81e84c293944292dfb05cf09048a5eb8bbfca40b Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Mon, 28 Dec 2020 18:03:14 -0500 Subject: [PATCH 114/459] Use execState instead of snd . runState --- src/ShellCheck/AnalyzerLib.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index a1e089a..f8e490e 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -270,7 +270,7 @@ executableFromShebang = shellFor -- This is used to populate parentMap in Parameters getParentTree :: Token -> Map.Map Id Token getParentTree t = - snd . snd $ runState (doStackAnalysis pre post t) ([], Map.empty) + snd $ execState (doStackAnalysis pre post t) ([], Map.empty) where pre t = modify (first ((:) t)) post t = do From e272fa04eed3e3f205e048a207d99e4cb011efcd Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Mon, 28 Dec 2020 18:04:32 -0500 Subject: [PATCH 115/459] Remove redundant bind and return --- src/ShellCheck/Parser.hs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 104e0a4..a17be91 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -209,8 +209,7 @@ startSpan = IncompleteInterval <$> getPosition endSpan (IncompleteInterval start) = do endPos <- getPosition - id <- getNextIdBetween start endPos - return id + getNextIdBetween start endPos getSpanPositionsFor m = do start <- getPosition From 953d9bc56dedba7a09829f2442204ae4b23074e1 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Mon, 28 Dec 2020 18:05:55 -0500 Subject: [PATCH 116/459] Remove unused helper stub --- src/ShellCheck/Parser.hs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index a17be91..48d1cfa 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -2088,10 +2088,6 @@ readSimpleCommand = called "simple command" $ do then action else getParser def cmd rest - cStyleComment cmd = - case cmd of - _ -> False - validateCommand cmd = case cmd of (T_NormalWord _ [T_Literal _ "//"]) -> commentWarning (getId cmd) From 2cfd1f27140d15132a031ef71a715e522a40837b Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Mon, 28 Dec 2020 18:10:47 -0500 Subject: [PATCH 117/459] Fuse maps --- shellcheck.hs | 2 +- src/ShellCheck/Checks/Commands.hs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/shellcheck.hs b/shellcheck.hs index f1757ef..d7e818d 100644 --- a/shellcheck.hs +++ b/shellcheck.hs @@ -507,7 +507,7 @@ ioInterface options files = do where find filename deflt = do sources <- findM ((allowable inputs) `andM` doesFileExist) $ - (adjustPath filename):(map ( filename) $ map adjustPath $ sourcePathFlag ++ sourcePathAnnotation) + (adjustPath filename):(map (( filename) . adjustPath) $ sourcePathFlag ++ sourcePathAnnotation) case sources of Nothing -> return deflt Just first -> return first diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 771083e..5be5a9a 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -951,7 +951,7 @@ checkCatastrophicRm = CommandCheck (Basename "rm") $ \t -> when (isRecursive t) $ mapM_ (mapM_ checkWord . braceExpand) $ arguments t where - isRecursive = any (`elem` ["r", "R", "recursive"]) . map snd . getAllFlags + isRecursive = any ((`elem` ["r", "R", "recursive"]) . snd) . getAllFlags checkWord token = case getLiteralString token of From fbb14d6b384a65ee16781966d9d240db9ed7b644 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 30 Dec 2020 19:24:14 -0800 Subject: [PATCH 118/459] Improve checks for = in command names (fixes #2102) --- CHANGELOG.md | 1 + src/ShellCheck/ASTLib.hs | 11 +- src/ShellCheck/Analytics.hs | 258 +++++++++++++++++++++++++++++++++++- src/ShellCheck/Parser.hs | 26 ++-- 4 files changed, 270 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0617ab0..596db82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ - POSIX/dash unsupported feature warnings now have individual SC3xxx codes - SC1090: A leading `$x/` or `$(x)/` is now treated as `./` when locating files - SC2154: Variables appearing in -z/-n tests are no longer considered unassigned +- SC2270-SC2285: Improved warnings about misused =, e.g. `${var}=42` ## v0.7.1 - 2020-04-04 diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index dcc9904..67d09f4 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -278,6 +278,12 @@ getUnquotedLiteral (T_NormalWord _ list) = str _ = Nothing getUnquotedLiteral _ = Nothing +isQuotes t = + case t of + T_DoubleQuoted {} -> True + T_SingleQuoted {} -> True + _ -> False + -- Get the last unquoted T_Literal in a word like "${var}foo"THIS -- or nothing if the word does not end in an unquoted literal. getTrailingUnquotedLiteral :: Token -> Maybe Token @@ -296,8 +302,11 @@ getTrailingUnquotedLiteral t = getLeadingUnquotedString :: Token -> Maybe String getLeadingUnquotedString t = case t of - T_NormalWord _ ((T_Literal _ s) : _) -> return s + T_NormalWord _ ((T_Literal _ s) : rest) -> return $ s ++ from rest _ -> Nothing + where + from ((T_Literal _ s):rest) = s ++ from rest + from _ = "" -- Maybe get the literal string of this token and any globs in it. getGlobOrLiteralString = getLiteralStringExt f diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 2fb1253..b7a444d 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -194,6 +194,8 @@ nodeChecks = [ ,checkBlatantRecursion ,checkBadTestAndOr ,checkAssignToSelf + ,checkEqualsInCommand + ,checkSecondArgIsComparison ] optionalChecks = map fst optionalTreeChecks @@ -277,15 +279,23 @@ checkUnqualifiedCommand str f t@(T_SimpleCommand id _ (cmd:rest)) | t `isUnqualifiedCommand` str = f cmd rest checkUnqualifiedCommand _ _ _ = return () +verifyCodes :: (Parameters -> Token -> Writer [TokenComment] ()) -> [Code] -> String -> Bool +verifyCodes f l s = codes == Just l + where + treeCheck = runNodeAnalysis f + comments = runAndGetComments treeCheck s + codes = map (cCode . tcComment) <$> comments checkNode f = producesComments (runNodeAnalysis f) producesComments :: (Parameters -> Token -> [TokenComment]) -> String -> Maybe Bool -producesComments f s = do +producesComments f s = not . null <$> runAndGetComments f s + +runAndGetComments f s = do let pr = pScript s prRoot pr let spec = defaultSpec pr let params = makeParameters spec - return . not . null $ + return $ filterByAnnotation spec params $ runList spec [f] @@ -364,6 +374,19 @@ replaceEnd id params n r = repPrecedence = depth, repInsertionPoint = InsertBefore } +replaceToken id params r = + let tp = tokenPositions params + (start, end) = tp Map.! id + depth = length $ getPath (parentMap params) (T_EOF id) + in + newReplacement { + repStartPos = start, + repEndPos = end, + repString = r, + repPrecedence = depth, + repInsertionPoint = InsertBefore + } + surroundWidth id params s = fixWith [replaceStart id params 0 s, replaceEnd id params 0 s] fixWith fixes = newFix { fixReplacements = fixes } @@ -1855,6 +1878,7 @@ prop_checkSpacefulness39= verifyNotTree checkSpacefulness "a=''\"\"''; b=x$a; ec prop_checkSpacefulness40= verifyNotTree checkSpacefulness "a=$((x+1)); echo $a" prop_checkSpacefulness41= verifyNotTree checkSpacefulness "exec $1 --flags" prop_checkSpacefulness42= verifyNotTree checkSpacefulness "run $1 --flags" +prop_checkSpacefulness43= verifyNotTree checkSpacefulness "$foo=42" data SpaceStatus = SpaceSome | SpaceNone | SpaceEmpty deriving (Eq) instance Semigroup SpaceStatus where @@ -1879,9 +1903,10 @@ checkSpacefulness params = checkSpacefulness' onFind params emit $ makeComment InfoC (getId token) 2223 "This default assignment may cause DoS due to globbing. Quote it." else - emit $ makeCommentWithFix InfoC (getId token) 2086 - "Double quote to prevent globbing and word splitting." - (addDoubleQuotesAround params token) + unless (quotesMayConflictWithSC2281 params token) $ + emit $ makeCommentWithFix InfoC (getId token) 2086 + "Double quote to prevent globbing and word splitting." + (addDoubleQuotesAround params token) isDefaultAssignment parents token = let modifier = getBracedModifier $ bracedString token in @@ -1896,14 +1921,25 @@ prop_checkSpacefulness4v= verifyTree checkVerboseSpacefulness "foo=3; foo=$(echo prop_checkSpacefulness8v= verifyTree checkVerboseSpacefulness "a=foo\\ bar; a=foo; rm $a" prop_checkSpacefulness28v = verifyTree checkVerboseSpacefulness "exec {n}>&1; echo $n" prop_checkSpacefulness36v = verifyTree checkVerboseSpacefulness "arg=$#; echo $arg" +prop_checkSpacefulness44v = verifyNotTree checkVerboseSpacefulness "foo=3; $foo=4" checkVerboseSpacefulness params = checkSpacefulness' onFind params where onFind spaces token name = - when (spaces == SpaceNone && name `notElem` specialVariablesWithoutSpaces) $ + when (spaces == SpaceNone + && name `notElem` specialVariablesWithoutSpaces + && not (quotesMayConflictWithSC2281 params token)) $ tell [makeCommentWithFix StyleC (getId token) 2248 "Prefer double quoting even when variables don't contain special characters." (addDoubleQuotesAround params token)] +-- Don't suggest quotes if this will instead be autocorrected +-- from $foo=bar to foo=bar. This is not pretty but ok. +quotesMayConflictWithSC2281 params t = + case getPath (parentMap params) t of + _ : T_NormalWord parentId (me:T_Literal _ ('=':_):_) : T_SimpleCommand _ _ (cmd:_) : _ -> + (getId t) == (getId me) && (parentId == getId cmd) + _ -> False + addDoubleQuotesAround params token = (surroundWidth (getId token) params "\"") checkSpacefulness' :: (SpaceStatus -> Token -> String -> Writer [TokenComment] ()) -> @@ -1978,8 +2014,9 @@ prop_CheckVariableBraces1 = verify checkVariableBraces "a='123'; echo $a" prop_CheckVariableBraces2 = verifyNot checkVariableBraces "a='123'; echo ${a}" prop_CheckVariableBraces3 = verifyNot checkVariableBraces "#shellcheck disable=SC2016\necho '$a'" prop_CheckVariableBraces4 = verifyNot checkVariableBraces "echo $* $1" +prop_CheckVariableBraces5 = verifyNot checkVariableBraces "$foo=42" checkVariableBraces params t@(T_DollarBraced id False l) - | name `notElem` unbracedVariables = + | name `notElem` unbracedVariables && not (quotesMayConflictWithSC2281 params t) = styleWithFix id 2250 "Prefer putting braces around variable references even when not strictly required." (fixFor t) @@ -4003,5 +4040,212 @@ checkAssignToSelf _ t = msg id = info id 2269 "This variable is assigned to itself, so the assignment does nothing." +prop_checkEqualsInCommand1a = verifyCodes checkEqualsInCommand [2277] "#!/bin/bash\n0='foo'" +prop_checkEqualsInCommand2a = verifyCodes checkEqualsInCommand [2278] "#!/bin/ksh \n$0='foo'" +prop_checkEqualsInCommand3a = verifyCodes checkEqualsInCommand [2279] "#!/bin/dash\n${0}='foo'" +prop_checkEqualsInCommand4a = verifyCodes checkEqualsInCommand [2280] "#!/bin/sh \n0='foo'" + +prop_checkEqualsInCommand1b = verifyCodes checkEqualsInCommand [2270] "1='foo'" +prop_checkEqualsInCommand2b = verifyCodes checkEqualsInCommand [2270] "${2}='foo'" + +prop_checkEqualsInCommand1c = verifyCodes checkEqualsInCommand [2271] "var$((n+1))=value" +prop_checkEqualsInCommand2c = verifyCodes checkEqualsInCommand [2271] "var${x}=value" +prop_checkEqualsInCommand3c = verifyCodes checkEqualsInCommand [2271] "var$((cmd))x='foo'" +prop_checkEqualsInCommand4c = verifyCodes checkEqualsInCommand [2271] "$(cmd)='foo'" + +prop_checkEqualsInCommand1d = verifyCodes checkEqualsInCommand [2273] "=======" +prop_checkEqualsInCommand2d = verifyCodes checkEqualsInCommand [2274] "======= Here =======" +prop_checkEqualsInCommand3d = verifyCodes checkEqualsInCommand [2275] "foo\n=42" + +prop_checkEqualsInCommand1e = verifyCodes checkEqualsInCommand [] "--foo=bar" +prop_checkEqualsInCommand2e = verifyCodes checkEqualsInCommand [] "$(cmd)'=foo'" +prop_checkEqualsInCommand3e = verifyCodes checkEqualsInCommand [2276] "var${x}/=value" +prop_checkEqualsInCommand4e = verifyCodes checkEqualsInCommand [2276] "${}=value" +prop_checkEqualsInCommand5e = verifyCodes checkEqualsInCommand [2276] "${#x}=value" + +prop_checkEqualsInCommand1f = verifyCodes checkEqualsInCommand [2281] "$var=foo" +prop_checkEqualsInCommand2f = verifyCodes checkEqualsInCommand [2281] "$a=$b" +prop_checkEqualsInCommand3f = verifyCodes checkEqualsInCommand [2281] "${var}=foo" +prop_checkEqualsInCommand4f = verifyCodes checkEqualsInCommand [2281] "${var[42]}=foo" +prop_checkEqualsInCommand5f = verifyCodes checkEqualsInCommand [2281] "$var+=foo" + +prop_checkEqualsInCommand1g = verifyCodes checkEqualsInCommand [2282] "411toppm=true" + +checkEqualsInCommand params originalToken = + case originalToken of + T_SimpleCommand _ _ (word:_) -> check word + _ -> return () + where + hasEquals t = + case t of + T_Literal _ s -> '=' `elem` s + _ -> False + + check t@(T_NormalWord _ list) | any hasEquals list = + case break hasEquals list of + (leading, (eq:_)) -> msg t (stripSinglePlus leading) eq + _ -> return () + check _ = return () + + -- This is a workaround for the parser adding + and = as separate literals + stripSinglePlus l = + case reverse l of + (T_Literal _ "+"):rest -> reverse rest + _ -> l + + positionalAssignmentRe = mkRegex "^[0-9][0-9]?=" + positionalMsg id = + err id 2270 "To assign positional parameters, use 'set -- first second ..' (or use [ ] to compare)." + indirectionMsg id = + err id 2271 "For indirection, use arrays, declare \"var$n=value\", or (for sh) read/eval." + badComparisonMsg id = + err id 2272 "Command name contains ==. For comparison, use [ \"$var\" = value ]." + conflictMarkerMsg id = + err id 2273 "Sequence of ===s found. Merge conflict or intended as a commented border?" + borderMsg id = + err id 2274 "Command name starts with ===. Intended as a commented border?" + prefixMsg id = + err id 2275 "Command name starts with =. Bad line break?" + genericMsg id = + err id 2276 "This is interpreted as a command name containing '='. Bad assignment or comparison?" + assign0Msg id bashfix = + case shellType params of + Bash -> errWithFix id 2277 "Use BASH_ARGV0 to assign to $0 in bash (or use [ ] to compare)." bashfix + Ksh -> err id 2278 "$0 can't be assigned in Ksh (but it does reflect the current function)." + Dash -> err id 2279 "$0 can't be assigned in Dash. This becomes a command name." + _ -> err id 2280 "$0 can't be assigned this way, and there is no portable alternative." + leadingNumberMsg id = + err id 2282 "Variable names can't start with numbers, so this is interpreted as a command." + + isExpansion t = + case t of + T_Arithmetic {} -> True + _ -> isQuoteableExpansion t + + isConflictMarker cmd = fromMaybe False $ do + str <- getUnquotedLiteral cmd + guard $ all (== '=') str + guard $ length str >= 4 && length str <= 12 -- Git uses 7 but who knows + return True + + mayBeVariableName l = fromMaybe False $ do + guard . not $ any isQuotes l + guard . not $ any willBecomeMultipleArgs l + str <- getLiteralStringExt (\_ -> Just "x") (T_NormalWord (Id 0) l) + return $ isVariableName str + + isLeadingNumberVar s = + let lead = takeWhile (/= '=') s + in not (null lead) && isDigit (head lead) + && all isVariableChar lead && not (all isDigit lead) + + msg cmd leading (T_Literal litId s) = do + -- There are many different cases, and the order of the branches matter. + case leading of + -- --foo=42 + [] | "-" `isPrefixOf` s -> -- There's SC2215 for these + return () + + -- ======Hello====== + [] | "=" `isPrefixOf` s -> + case originalToken of + T_SimpleCommand _ [] [word] | isConflictMarker word -> + conflictMarkerMsg (getId originalToken) + _ | "===" `isPrefixOf` s -> borderMsg (getId originalToken) + _ -> prefixMsg (getId cmd) + + -- $var==42 + _ | "==" `isInfixOf` s -> + badComparisonMsg (getId cmd) + + -- ${foo[x]}=42 and $foo=42 + [T_DollarBraced id braced l] | "=" `isPrefixOf` s -> do + let variableStr = concat $ oversimplify l + let variableReference = getBracedReference variableStr + let variableModifier = getBracedModifier variableStr + let isPlain = isVariableName variableStr + let isPositional = all isDigit variableStr + + let isArray = variableReference /= "" + && "[" `isPrefixOf` variableModifier + && "]" `isSuffixOf` variableModifier + + case () of + -- $foo=bar should already have caused a parse-time SC1066 + -- _ | not braced && isPlain -> + -- return () + + _ | variableStr == "" -> -- Don't try to fix ${}=foo + genericMsg (getId cmd) + + -- $#=42 or ${#var}=42 + _ | "#" `isPrefixOf` variableStr -> + genericMsg (getId cmd) + + -- ${0}=42 + _ | variableStr == "0" -> + assign0Msg id $ fixWith [replaceToken id params "BASH_ARGV0"] + + -- $2=2 + _ | isPositional -> + positionalMsg id + + _ | isArray || isPlain -> + errWithFix id 2281 + ("Don't use " ++ (if braced then "${}" else "$") ++ " on the left side of assignments.") $ + fixWith $ + if braced + then [ replaceStart id params 2 "", replaceEnd id params 1 "" ] + else [ replaceStart id params 1 "" ] + + _ -> indirectionMsg id + + -- 2=42 + [] | s `matches` positionalAssignmentRe -> + if "0=" `isPrefixOf` s + then + assign0Msg litId $ fixWith [replaceStart litId params 1 "BASH_ARGV0"] + else + positionalMsg litId + + -- 9foo=42 + [] | isLeadingNumberVar s -> + leadingNumberMsg (getId cmd) + + -- var${foo}x=42 + (_:_) | mayBeVariableName leading && (all isVariableChar $ takeWhile (/= '=') s) -> + indirectionMsg (getId cmd) + + _ -> genericMsg (getId cmd) + + +prop_checkSecondArgIsComparison1 = verify checkSecondArgIsComparison "foo = $bar" +prop_checkSecondArgIsComparison2 = verify checkSecondArgIsComparison "$foo = $bar" +prop_checkSecondArgIsComparison3 = verify checkSecondArgIsComparison "2f == $bar" +prop_checkSecondArgIsComparison4 = verify checkSecondArgIsComparison "'var' =$bar" +prop_checkSecondArgIsComparison5 = verify checkSecondArgIsComparison "foo ='$bar'" +prop_checkSecondArgIsComparison6 = verify checkSecondArgIsComparison "$foo =$bar" +prop_checkSecondArgIsComparison7 = verify checkSecondArgIsComparison "2f ==$bar" +prop_checkSecondArgIsComparison8 = verify checkSecondArgIsComparison "'var' =$bar" +prop_checkSecondArgIsComparison9 = verify checkSecondArgIsComparison "var += $(foo)" +prop_checkSecondArgIsComparison10 = verify checkSecondArgIsComparison "var +=$(foo)" +checkSecondArgIsComparison _ t = + case t of + T_SimpleCommand _ _ (lhs:arg:_) -> sequence_ $ do + argString <- getLeadingUnquotedString arg + case argString of + '=':'=':'=':'=':_ -> Nothing -- Don't warn about `echo ======` and such + '+':'=':_ -> + return $ err (getId t) 2285 $ + "Remove spaces around += to assign (or quote '+=' if literal)." + '=':'=':_ -> + return $ err (getId t) 2284 $ + "Use [ x = y ] to compare values (or quote '==' if literal)." + '=':_ -> + return $ err (getId t) 2283 $ + "Use [ ] to compare values, or remove spaces around = to assign (or quote '=' if literal)." + _ -> Nothing + _ -> return () + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index dbadc7c..5634d95 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -2850,15 +2850,13 @@ readAssignmentWordExt lenient = called "variable assignment" $ do (id, variable, op, indices) <- try $ do start <- startSpan pos <- getPosition + -- Check for a leading $ at parse time, to warn for $foo=(bar) which + -- would otherwise cause a parse failure so it can't be checked later. leadingDollarPos <- if lenient then optionMaybe $ getSpanPositionsFor (char '$') else return Nothing variable <- readVariableName - middleDollarPos <- - if lenient - then optionMaybe $ getSpanPositionsFor readNormalDollar - else return Nothing indices <- many readArrayIndex hasLeftSpace <- fmap (not . null) spacing opStart <- getPosition @@ -2866,20 +2864,12 @@ readAssignmentWordExt lenient = called "variable assignment" $ do op <- readAssignmentOp opEnd <- getPosition - when (isJust leadingDollarPos || isJust middleDollarPos || hasLeftSpace) $ do - sequence_ $ do - (l, r) <- leadingDollarPos - return $ parseProblemAtWithEnd l r ErrorC 1066 "Don't use $ on the left side of assignments." - sequence_ $ do - (l, r) <- middleDollarPos - return $ parseProblemAtWithEnd l r ErrorC 1067 "For indirection, use arrays, declare \"var$n=value\", or (for sh) read/eval." - when hasLeftSpace $ do - parseProblemAtWithEnd opStart opEnd ErrorC 1068 $ - "Don't put spaces around the " - ++ (if op == Append - then "+= when appending" - else "= in assignments") - ++ " (or quote to make it literal)." + when (isJust leadingDollarPos || hasLeftSpace) $ do + hasParen <- isFollowedBy (spacing >> char '(') + when hasParen $ + sequence_ $ do + (l, r) <- leadingDollarPos + return $ parseProblemAtWithEnd l r ErrorC 1066 "Don't use $ on the left side of assignments." -- Fail so that this is not parsed as an assignment. fail "" From 5fbaae2bb3f9f26912e55d41894f592be82b4ec1 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 30 Dec 2020 20:55:18 -0800 Subject: [PATCH 119/459] Don't treat ${!x@} as reference of x (fixes #2116) --- src/ShellCheck/AnalyzerLib.hs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 194bf18..bc8bc3d 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -846,6 +846,7 @@ prop_getBracedReference8 = getBracedReference "foo-bar" == "foo" prop_getBracedReference9 = getBracedReference "foo:-bar" == "foo" prop_getBracedReference10= getBracedReference "foo: -1" == "foo" prop_getBracedReference11= getBracedReference "!os*" == "" +prop_getBracedReference11b= getBracedReference "!os@" == "" prop_getBracedReference12= getBracedReference "!os?bar**" == "" prop_getBracedReference13= getBracedReference "foo[bar]" == "foo" getBracedReference s = fromMaybe s $ @@ -864,7 +865,7 @@ getBracedReference s = fromMaybe s $ nameExpansion ('!':next:rest) = do -- e.g. ${!foo*bar*} guard $ isVariableChar next -- e.g. ${!@} first <- find (not . isVariableChar) rest - guard $ first `elem` "*?" + guard $ first `elem` "*?@" return "" nameExpansion _ = Nothing From 9584266a8b455c6d6d9cadc45a8647b1fef8ad74 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 31 Dec 2020 12:28:48 -0800 Subject: [PATCH 120/459] Escape control characters when adding user data to messages --- src/ShellCheck/ASTLib.hs | 32 +++++++++++++++++++++++++++++++ src/ShellCheck/Analytics.hs | 4 ++-- src/ShellCheck/Checks/Commands.hs | 2 +- src/ShellCheck/Parser.hs | 10 +++++----- 4 files changed, 40 insertions(+), 8 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 67d09f4..d9ee8b1 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -29,6 +29,7 @@ import Data.Functor.Identity import Data.List import Data.Maybe import qualified Data.Map as Map +import Numeric (showHex) arguments (T_SimpleCommand _ _ (cmd:args)) = args @@ -367,6 +368,37 @@ getLiteralStringExt more = g -- Is this token a string literal? isLiteral t = isJust $ getLiteralString t +-- Escape user data for messages. +-- Messages generally avoid repeating user data, but sometimes it's helpful. +e4m = escapeForMessage +escapeForMessage :: String -> String +escapeForMessage str = concatMap f str + where + f '\\' = "\\\\" + f '\n' = "\\n" + f '\r' = "\\r" + f '\t' = "\\t" + f '\x1B' = "\\e" + f c = + if shouldEscape c + then + if ord c < 256 + then "\\x" ++ (pad0 2 $ toHex c) + else "\\U" ++ (pad0 4 $ toHex c) + else [c] + + shouldEscape c = + (not $ isPrint c) + || (not (isAscii c) && not (isLetter c)) + + pad0 :: Int -> String -> String + pad0 n s = + let l = length s in + if l < n + then (replicate (n-l) '0') ++ s + else s + toHex :: Char -> String + toHex c = map toUpper $ showHex (ord c) "" -- Turn a NormalWord like foo="bar $baz" into a series of constituent elements like [foo=,bar ,$baz] getWordParts (T_NormalWord _ l) = concatMap getWordParts l diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index b7a444d..102b908 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1759,7 +1759,7 @@ checkSshHereDoc _ (T_Redirecting _ redirs cmd) hasVariables = mkRegex "[`$]" checkHereDoc (T_FdRedirect _ _ (T_HereDoc id _ Unquoted token tokens)) | not (all isConstant tokens) = - warn id 2087 $ "Quote '" ++ token ++ "' to make here document expansions happen on the server side rather than on the client." + warn id 2087 $ "Quote '" ++ (e4m token) ++ "' to make here document expansions happen on the server side rather than on the client." checkHereDoc _ = return () checkSshHereDoc _ _ = return () @@ -2694,7 +2694,7 @@ checkUnpassedInFunctions params root = suggestParams (name, _, thing) = info (getId thing) 2119 $ - "Use " ++ name ++ " \"$@\" if function's $1 should mean script's $1." + "Use " ++ (e4m name) ++ " \"$@\" if function's $1 should mean script's $1." warnForDeclaration func name = warn (getId func) 2120 $ name ++ " references arguments, but none are ever passed." diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index aa4edd4..36f32a5 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -900,7 +900,7 @@ checkWhileGetoptsCase = CommandCheck (Exactly "getopts") f notRequested = Map.difference handledMap requestedMap warnUnhandled optId caseId str = - warn caseId 2213 $ "getopts specified -" ++ str ++ ", but it's not handled by this 'case'." + warn caseId 2213 $ "getopts specified -" ++ (e4m str) ++ ", but it's not handled by this 'case'." warnRedundant (Just str, expr) | str `notElem` ["*", ":", "?"] = diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 5634d95..b70e4f4 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -1904,14 +1904,14 @@ readPendingHereDocs = do debugHereDoc tokenId endToken doc | endToken `isInfixOf` doc = let lookAt line = when (endToken `isInfixOf` line) $ - parseProblemAtId tokenId ErrorC 1042 ("Close matches include '" ++ line ++ "' (!= '" ++ endToken ++ "').") + parseProblemAtId tokenId ErrorC 1042 ("Close matches include '" ++ (e4m line) ++ "' (!= '" ++ (e4m endToken) ++ "').") in do - parseProblemAtId tokenId ErrorC 1041 ("Found '" ++ endToken ++ "' further down, but not on a separate line.") + parseProblemAtId tokenId ErrorC 1041 ("Found '" ++ (e4m endToken) ++ "' further down, but not on a separate line.") mapM_ lookAt (lines doc) | map toLower endToken `isInfixOf` map toLower doc = - parseProblemAtId tokenId ErrorC 1043 ("Found " ++ endToken ++ " further down, but with wrong casing.") + parseProblemAtId tokenId ErrorC 1043 ("Found " ++ (e4m endToken) ++ " further down, but with wrong casing.") | otherwise = - parseProblemAtId tokenId ErrorC 1044 ("Couldn't find end token `" ++ endToken ++ "' in the here document.") + parseProblemAtId tokenId ErrorC 1044 ("Couldn't find end token `" ++ (e4m endToken) ++ "' in the here document.") readFilename = readNormalWord @@ -3168,7 +3168,7 @@ readConfigFile filename = do let line = "line " ++ (show . sourceLine $ errorPos err) suggestion = getStringFromParsec $ errorMessages err in - "Failed to process " ++ filename ++ ", " ++ line ++ ": " + "Failed to process " ++ (e4m filename) ++ ", " ++ line ++ ": " ++ suggestion prop_readConfigKVs1 = isOk readConfigKVs "disable=1234" From 2e5c56b27034492134be1538c1e1e6a533ca791a Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 31 Dec 2020 13:19:14 -0800 Subject: [PATCH 121/459] Parse heredocs correctly with carriage returns (fixes #2103) --- CHANGELOG.md | 1 + src/ShellCheck/Parser.hs | 10 ++++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 596db82..ff36bb2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ is still purely cosmetic and does not allow ShellCheck to continue. - Improved error reporting for trailing tokens after ]/]] and compound commands - `#!/usr/bin/env -S shell` is now handled correctly +- Here docs with \r are now parsed correctly and give better warnings ### Changed - Assignments are now parsed to spec, without leniency for leading $ or spaces diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index b70e4f4..afe6262 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -123,8 +123,10 @@ readUnicodeQuote = do return $ T_Literal id [c] carriageReturn = do - parseNote ErrorC 1017 "Literal carriage return. Run script through tr -d '\\r' ." + pos <- getPosition char '\r' + parseProblemAt pos ErrorC 1017 "Literal carriage return. Run script through tr -d '\\r' ." + return '\r' almostSpace = choice [ @@ -1761,6 +1763,8 @@ prop_readHereDoc17= isWarning readScript "cat <<- ' foo'\nbar\n foo\n foo\n" prop_readHereDoc18= isOk readScript "cat <<'\"foo'\nbar\n\"foo\n" prop_readHereDoc20= isWarning readScript "cat << foo\n foo\n()\nfoo\n" prop_readHereDoc21= isOk readScript "# shellcheck disable=SC1039\ncat << foo\n foo\n()\nfoo\n" +prop_readHereDoc22 = isWarning readScript "cat << foo\r\ncow\r\nfoo\r\n" +prop_readHereDoc23 = isNotOk readScript "cat << foo \r\ncow\r\nfoo\r\n" readHereDoc = called "here document" $ do pos <- getPosition try $ string "<<" @@ -1789,7 +1793,9 @@ readHereDoc = called "here document" $ do -- Fun fact: bash considers << foo"" quoted, but not << <("foo"). readToken = do str <- readStringForParser readNormalWord - return $ unquote str + -- A here doc actually works with \r\n because the \r becomes part of the token + crstr <- (carriageReturn >> (return $ str ++ "\r")) <|> return str + return $ unquote crstr readPendingHereDocs = do docs <- popPendingHereDocs From c5756760cbadf48e501896641f20f1bda9f17420 Mon Sep 17 00:00:00 2001 From: freddii Date: Tue, 5 Jan 2021 14:08:03 +0100 Subject: [PATCH 122/459] fixed typing mistakes in changelog --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff36bb2..2d7a1ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -164,7 +164,7 @@ - SC2204/SC2205: Warn about `( -z foo )` and `( foo -eq bar )` - SC2200/SC2201: Warn about brace expansion in [/[[ - SC2198/SC2199: Warn about arrays in [/[[ -- SC2196/SC2197: Warn about deprected egrep/fgrep +- SC2196/SC2197: Warn about deprecated egrep/fgrep - SC2195: Warn about unmatchable case branches - SC2194: Warn about constant 'case' statements - SC2193: Warn about `[[ file.png == *.mp3 ]]` and other unmatchables @@ -181,7 +181,7 @@ ### Fixed - `-c` no longer suggested when using `grep -o | wc` - Comments and whitespace are now allowed before filewide directives -- Here doc delimters with esoteric quoting like `foo""` are now handled +- Here doc delimiters with esoteric quoting like `foo""` are now handled - SC2095 about `ssh` in while read loops is now suppressed when using `-n` - `%(%Y%M%D)T` now recognized as a single formatter in `printf` checks - `grep -F` now suppresses regex related suggestions From dff8f9492a153b4ad8ac7d085136ce532e8ea081 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 5 Jan 2021 10:07:39 -0800 Subject: [PATCH 123/459] Improve SC2283 message and position --- src/ShellCheck/Analytics.hs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 102b908..8ab7f12 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -4236,16 +4236,22 @@ checkSecondArgIsComparison _ t = case argString of '=':'=':'=':'=':_ -> Nothing -- Don't warn about `echo ======` and such '+':'=':_ -> - return $ err (getId t) 2285 $ + return $ err (headId t) 2285 $ "Remove spaces around += to assign (or quote '+=' if literal)." '=':'=':_ -> return $ err (getId t) 2284 $ "Use [ x = y ] to compare values (or quote '==' if literal)." '=':_ -> - return $ err (getId t) 2283 $ - "Use [ ] to compare values, or remove spaces around = to assign (or quote '=' if literal)." + return $ err (headId arg) 2283 $ + "Remove spaces around = to assign (or use [ ] to compare, or quote '=' if literal)." _ -> Nothing _ -> return () + where + -- We don't pinpoint exactly, but this helps cases like foo =$bar + headId t = + case t of + T_NormalWord _ (x:_) -> getId x + _ -> getId t return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) From 99e9d5c54b0a3188b9c2b2ae09276429eea39b78 Mon Sep 17 00:00:00 2001 From: Kir Kolyshkin Date: Wed, 27 Jan 2021 16:21:44 -0800 Subject: [PATCH 124/459] Whitelist podman for SC2016 about '$var' Same as 08d2eef4111afde4 but for podman. Fixes https://github.com/koalaman/shellcheck/issues/2057 Signed-off-by: Kir Kolyshkin --- src/ShellCheck/Analytics.hs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 8ab7f12..a9ce515 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1015,6 +1015,7 @@ checkSingleQuotedVariables params t@(T_SingleQuoted id s) = ,"alias" ,"sudo" -- covering "sudo sh" and such ,"docker" -- like above + ,"podman" ,"dpkg-query" ,"jq" -- could also check that user provides --arg ,"rename" From 2e59eba6ebcafa029076a37cdea09521860ee01d Mon Sep 17 00:00:00 2001 From: Austin English Date: Fri, 5 Feb 2021 19:56:44 -0600 Subject: [PATCH 125/459] add support for `/bin/busybox sh` shebang --- src/ShellCheck/Analytics.hs | 6 ++++++ src/ShellCheck/Parser.hs | 4 +++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 8500a7b..cb02eae 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -590,6 +590,12 @@ prop_checkShebang9 = verifyNotTree checkShebang "# shellcheck shell=sh\ntrue" prop_checkShebang10= verifyNotTree checkShebang "#!foo\n# shellcheck shell=sh ignore=SC2239\ntrue" prop_checkShebang11= verifyTree checkShebang "#!/bin/sh/\ntrue" prop_checkShebang12= verifyTree checkShebang "#!/bin/sh/ -xe\ntrue" +prop_checkShebang13= verifyTree checkShebang "#!/bin/busybox sh" +prop_checkShebang14= verifyTree checkShebang "#!/bin/busybox sh\n# shellcheck shell=sh\n" +prop_checkShebang15= verifyNotTree checkShebang "#!/bin/busybox sh\n# shellcheck shell=dash\n" +prop_checkShebang16= verifyTree checkShebang "#!/bin/busybox ash" +prop_checkShebang17= verifyNotTree checkShebang "#!/bin/busybox ash\n# shellcheck shell=dash\n" +prop_checkShebang18= verifyNotTree checkShebang "#!/bin/busybox ash\n# shellcheck shell=sh\n" checkShebang params (T_Annotation _ list t) = if any isOverride list then [] else checkShebang params t where diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 03865af..a54bbc3 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -3238,7 +3238,9 @@ readScriptFile sourced = do (first:second:_) -> if basename first == "env" then second - else basename first + else if basename first == "busybox" + then second + else basename first verifyShebang pos s = do case isValidShell s of From 8bb5e01401ade0962727d9001ba34cfbf10937d6 Mon Sep 17 00:00:00 2001 From: Claudio Bley Date: Fri, 12 Feb 2021 10:44:18 +0100 Subject: [PATCH 126/459] Allow `env` to have flags and variables in shebang The `env` command has a `-S,--split-string` option which enables having arguments for the command in a shebang. Also, one could use variable assignments for the command since `env` treats only the first word without a `=` character as the command to run. Fixes #2143. --- src/ShellCheck/Parser.hs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 03865af..9f5208e 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -3190,6 +3190,7 @@ prop_readScript2 = isWarning readScript "#!/bin/bash\r\necho hello world\n" prop_readScript3 = isWarning readScript "#!/bin/bash\necho hello\xA0world" prop_readScript4 = isWarning readScript "#!/usr/bin/perl\nfoo=(" prop_readScript5 = isOk readScript "#!/bin/bash\n#This is an empty script\n\n" +prop_readScript6 = isOk readScript "#!/usr/bin/env -S X=FOO bash\n#This is an empty script\n\n" readScriptFile sourced = do start <- startSpan pos <- getPosition @@ -3231,13 +3232,14 @@ readScriptFile sourced = do where basename s = reverse . takeWhile (/= '/') . reverse $ s + skipFlags = dropWhile ("-" `isPrefixOf`) getShell sb = case words sb of [] -> "" [x] -> basename x - (first:second:_) -> + (first:args) -> if basename first == "env" - then second + then fromMaybe "" $ find (notElem '=') $ skipFlags args else basename first verifyShebang pos s = do From d6bb8fc0d8aa40e72b2b894b16d7eacdafcf94a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Moritz=20R=C3=B6hrich?= Date: Sun, 14 Feb 2021 19:13:29 +0100 Subject: [PATCH 127/459] Error on backslash in comment #2132 - Report error in case of a backspace in a comment Backspaces in comments are no good. In most cases they are the result of commenting out a longer line, that was broken down. This usually results in the shell treating the following lines as their own commands on their own lines instead of as parts of the longer, broken down line. --- src/ShellCheck/Parser.hs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 03865af..a999b23 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -1039,9 +1039,14 @@ readComment = do unexpecting "shellcheck annotation" readAnnotationPrefix readAnyComment +prop_readAnyComment = isOk readAnyComment "# Comment" +prop_readAnyComment1 = not $ isOk readAnyComment "# Comment \\\n" readAnyComment = do char '#' - many $ noneOf "\r\n" + comment <- many $ noneOf "\\\r\n" + bs <- many $ oneOf "\\" + unless (null bs) (fail "Backslash in or directly after comment") + return comment prop_readNormalWord = isOk readNormalWord "'foo'\"bar\"{1..3}baz$(lol)" prop_readNormalWord2 = isOk readNormalWord "foo**(foo)!!!(@@(bar))" From b9b6975bfa8d57dc5c332587f5e46aec0ea9fb74 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 11 Feb 2021 17:18:43 -0800 Subject: [PATCH 128/459] Brand New Build! Features Linux x86_64 docker builds for all archs --- .compile_binaries | 73 ------------------- .github/workflows/build.yml | 124 ++++++++++++++++++++++++++++++++ .github_deploy | 35 +-------- .multi_arch_docker | 9 +-- .prepare_deploy | 36 ++++++---- .travis.yml | 97 ------------------------- Dockerfile | 29 -------- build/README.md | 13 ++++ build/build_builder | 12 ++++ build/darwin.x86_64/Dockerfile | 31 ++++++++ build/darwin.x86_64/build | 14 ++++ build/darwin.x86_64/tag | 1 + build/linux.aarch64/Dockerfile | 30 ++++++++ build/linux.aarch64/build | 15 ++++ build/linux.aarch64/tag | 1 + build/linux.armv6hf/Dockerfile | 59 +++++++++++++++ build/linux.armv6hf/build | 16 +++++ build/linux.armv6hf/tag | 1 + build/linux.x86_64/Dockerfile | 26 +++++++ build/linux.x86_64/build | 15 ++++ build/linux.x86_64/tag | 1 + build/run_builder | 30 ++++++++ build/windows.x86_64/Dockerfile | 27 +++++++ build/windows.x86_64/build | 19 +++++ build/windows.x86_64/tag | 1 + 25 files changed, 461 insertions(+), 254 deletions(-) delete mode 100755 .compile_binaries create mode 100644 .github/workflows/build.yml delete mode 100644 .travis.yml delete mode 100644 Dockerfile create mode 100644 build/README.md create mode 100755 build/build_builder create mode 100644 build/darwin.x86_64/Dockerfile create mode 100755 build/darwin.x86_64/build create mode 100644 build/darwin.x86_64/tag create mode 100644 build/linux.aarch64/Dockerfile create mode 100755 build/linux.aarch64/build create mode 100644 build/linux.aarch64/tag create mode 100644 build/linux.armv6hf/Dockerfile create mode 100755 build/linux.armv6hf/build create mode 100644 build/linux.armv6hf/tag create mode 100644 build/linux.x86_64/Dockerfile create mode 100755 build/linux.x86_64/build create mode 100644 build/linux.x86_64/tag create mode 100755 build/run_builder create mode 100644 build/windows.x86_64/Dockerfile create mode 100755 build/windows.x86_64/build create mode 100644 build/windows.x86_64/tag diff --git a/.compile_binaries b/.compile_binaries deleted file mode 100755 index 95939ae..0000000 --- a/.compile_binaries +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/bash - -build_linux() { - # Linux Docker image - name="$DOCKER_BASE" - DOCKER_BUILDS="$DOCKER_BUILDS $name" - docker build -t "$name:current" . - docker run "$name:current" --version - printf '%s\n' "#!/bin/sh" "echo 'hello world'" > myscript - docker run -v "$PWD:/mnt" "$name:current" myscript - - # Copy static executable from docker image - id=$(docker create "$name:current") - docker cp "$id:/bin/shellcheck" "shellcheck" - docker rm "$id" - ls -l shellcheck - ./shellcheck myscript - for tag in $TAGS - do - cp "shellcheck" "deploy/shellcheck-$tag.linux-x86_64"; - done -} - -build_aarch64() { - # Linux aarch64 static executable - docker run -v "$PWD:/mnt" koalaman/aarch64-builder 'buildsc' - for tag in $TAGS - do - cp "shellcheck" "deploy/shellcheck-$tag.linux-aarch64" - done -} - - -build_armv6hf() { - # Linux armv6hf static executable - docker run -v "$PWD:/mnt" koalaman/armv6hf-builder -c 'compile-shellcheck' - for tag in $TAGS - do - cp "shellcheck" "deploy/shellcheck-$tag.linux-armv6hf"; - done -} - -build_windows() { - # Windows .exe - docker run -v "$PWD:/appdata" koalaman/winghc cuib - for tag in $TAGS - do - cp "dist/build/ShellCheck/shellcheck.exe" "deploy/shellcheck-$tag.exe"; - done -} - -build_osx() { - # Darwin x86_64 executable - brew update - brew install cabal-install pandoc gnu-tar - sudo ln -sf /usr/local/bin/gsha512sum /usr/local/bin/sha512sum - sudo ln -sf /usr/local/bin/gtar /usr/local/bin/tar - export PATH="/usr/local/bin:$PATH" - - cabal update - cabal install --dependencies-only - cabal build shellcheck - - # Cabal 3 no longer has a predictable output path - path="$(find . -name 'shellcheck' -type f -perm +111)" - [[ -e "$path" ]] - - for tag in $TAGS - do - cp "$path" "deploy/shellcheck-$tag.darwin-x86_64"; - done -} - diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..cd9fca0 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,124 @@ +name: Build Lol + +# Run this workflow every time a new commit pushed to your repository +on: push + +jobs: + package_source: + name: Package Source Code + runs-on: ubuntu-latest + steps: + - name: Install Dependencies + run: | + sudo apt-get update + sudo apt-mark manual ghc # Don't bother installing ghc just to tar up source + sudo apt-get install cabal-install + + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Package Source + run: | + mkdir source + cabal sdist + mv dist/*.tar.gz source/source.tar.gz + + - name: Deduce tags + run: | + exec > source/tags + echo "latest" + if tag=$(git describe --exact-match --tags) + then + echo "stable" + echo "$tag" + fi + + - name: Upload artifact + uses: actions/upload-artifact@v2 + with: + name: source + path: source/ + + build_source: + name: Build Source Code + needs: package_source + strategy: + matrix: + build: [linux.x86_64, linux.aarch64, linux.armv6hf, darwin.x86_64, windows.x86_64] + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Download artifacts + uses: actions/download-artifact@v2 + + - name: Build source + run: | + mkdir -p bin + mkdir -p bin/${{matrix.build}} + ( cd bin && ../build/run_builder ../source/source.tar.gz ../build/${{matrix.build}} ) + + - name: Upload artifact + uses: actions/upload-artifact@v2 + with: + name: bin + path: bin/ + + package_binary: + name: Package Binaries + needs: build_source + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Download artifacts + uses: actions/download-artifact@v2 + + - name: Work around GitHub permissions bug + run: chmod +x bin/*/shellcheck* + + - name: Package binaries + run: | + export TAGS="$(cat source/tags)" + mkdir -p deploy + cp -r bin/* deploy + cd deploy + ../.prepare_deploy + rm -rf */ README* LICENSE* + + - name: Upload artifact + uses: actions/upload-artifact@v2 + with: + name: deploy + path: deploy/ + + deploy: + name: Deploy binaries + needs: package_binary + runs-on: ubuntu-latest + environment: Deploy + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Download artifacts + uses: actions/download-artifact@v2 + + - name: Upload to GitHub + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + export TAGS="$(cat source/tags)" + ./.github_deploy + + - name: Upload to Docker Hub + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + DOCKER_EMAIL: ${{ secrets.DOCKER_EMAIL }} + DOCKER_BASE: ${{ secrets.DOCKER_USERNAME }}/shellcheck + run: | + export TAGS="$(cat source/tags)" + ( source ./.multi_arch_docker && set -eux && multi_arch_docker::main ) diff --git a/.github_deploy b/.github_deploy index e5d6c3e..3de0ac2 100755 --- a/.github_deploy +++ b/.github_deploy @@ -2,39 +2,10 @@ set -x shopt -s extglob -if [[ "$TRAVIS_SECURE_ENV_VARS" != "true" ]] -then - echo >&2 "Missing TRAVIS_SECURE_ENV_VARS. Skipping GitHub deployment." - exit 0 -fi - -install_deps() { - version="2.7.0" # 2.14.1 fails to overwrite duplicates - case "$(uname)" in - Linux) - sudo apt-get update - sudo apt-get install curl - curl -L "https://github.com/github/hub/releases/download/v$version/hub-linux-amd64-$version.tgz" | tar xvz --strip-components=1 "hub-linux-amd64-$version/bin/hub" - ;; - Darwin) - curl -L "https://github.com/github/hub/releases/download/v$version/hub-darwin-amd64-$version.tgz" | tar xvz --strip-components=1 "hub-darwin-amd64-$version/bin/hub" - ;; - *) - echo "Unknown: $(uname)" - exit 1 - ;; - esac - - hub_path="$PWD/bin/hub" - hub() { - "$hub_path" "$@" - } -} -install_deps - export EDITOR="touch" # Sanity check +gh --version || exit 1 hub release show latest || exit 1 for tag in $TAGS @@ -51,8 +22,8 @@ do do [[ $file == *.@(xz|gz|zip) ]] || continue [[ $file == *"$tag"* ]] || continue - files+=(-a "$file") + files+=("$file") done - hub release edit "${files[@]}" "$tag" || exit 1 + gh release upload "$tag" "${files[@]}" --clobber || exit 1 done diff --git a/.multi_arch_docker b/.multi_arch_docker index 294fed2..a9f7401 100755 --- a/.multi_arch_docker +++ b/.multi_arch_docker @@ -1,12 +1,6 @@ #!/bin/bash # This script builds and deploys multi-architecture docker images from the -# binaries previously built and deployed to GCS by the Travis pipeline. - -if [[ "$TRAVIS_SECURE_ENV_VARS" != "true" ]] -then - echo >&2 "Missing TRAVIS_SECURE_ENV_VARS. Skipping Docker builds." - exit 0 -fi +# binaries previously built and deployed to GitHub. function multi_arch_docker::install_docker_buildx() { # Install up-to-date version of docker, with buildx support. @@ -108,6 +102,5 @@ function multi_arch_docker::main() { multi_arch_docker::install_docker_buildx multi_arch_docker::login_to_docker_hub multi_arch_docker::build_and_push_all - set +x multi_arch_docker::test_all } diff --git a/.prepare_deploy b/.prepare_deploy index a2ce361..9f39912 100755 --- a/.prepare_deploy +++ b/.prepare_deploy @@ -1,8 +1,9 @@ #!/bin/bash -# This script packages up Travis compiled binaries +# This script packages up compiled binaries set -ex shopt -s nullglob extglob -cd deploy + +ls -l cp ../LICENSE LICENSE.txt sed -e $'s/$/\r/' > README.txt << END @@ -22,26 +23,31 @@ This binary was compiled on $(date -u). $(git log -n 3) END -for file in ./*.exe +for dir in */ do - zip "${file%.*}.zip" README.txt LICENSE.txt "$file" + cp LICENSE.txt README.txt "$dir" done -for file in *.{linux,darwin}-* -do - base="${file%.*}" - ext="${file##*.}" - os="${ext%-*}" - arch="${ext##*-}" - cp "$file" "shellcheck" - tar -cJf "$base.$os.$arch.tar.xz" --transform="s:^:$base/:" README.txt LICENSE.txt shellcheck - rm "shellcheck" -done +echo "Tags are $TAGS" -rm !(*.xz|*.zip) +for tag in $TAGS +do + + for dir in windows.*/ + do + ( cd "$dir" && zip "../shellcheck-$tag.zip" * ) + done + + for dir in {linux,darwin}.*/ + do + base="${dir%/}" + ( cd "$dir" && tar -cJf "../shellcheck-$tag.$base.tar.xz" --transform="s:^:shellcheck-$tag/:" * ) + done +done for file in ./* do + [[ -f "$file" ]] || continue sha512sum "$file" > "$file.sha512sum" done diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index d7d202b..0000000 --- a/.travis.yml +++ /dev/null @@ -1,97 +0,0 @@ -language: shell -os: linux - -services: - - docker - -jobs: - include: - - stage: Build - # This must weirdly not have a dash, otherwise an empty job is created - env: BUILD=linux - workspaces: - create: - name: ws-linux - paths: deploy - - env: BUILD=windows - workspaces: - create: - name: ws-windows - paths: deploy - - env: BUILD=armv6hf - workspaces: - create: - name: ws-armv6hf - paths: deploy - - env: BUILD=aarch64 - workspaces: - create: - name: ws-aarch64 - paths: deploy - - env: BUILD=osx - os: osx - workspaces: - create: - name: ws-osx - paths: deploy - - - stage: Upload Artifacts to GitHub - workspaces: - use: - - ws-osx - - ws-linux - - ws-armv6hf - - ws-aarch64 - - ws-windows - script: - - ls -la ${CASHER_DIR}/ || true - # Kludge broken TravisCI workspaces - - tar -xvf ${CASHER_DIR}/ws-osx-fetch.tgz --strip-components=5 - - ls -la deploy - - ./.github_deploy - - - stage: Deploy docker image - # Deploy only for pushes to master branch, not other branches, not PRs. - if: type = push - script: - - source ./.multi_arch_docker - - set -ex; multi_arch_docker::main; set +x - -# This is in global context and runs for every stage that doesn't override it. -before_install: | - DOCKER_BASE="$DOCKER_USERNAME/shellcheck" - DOCKER_BUILDS="" - export TAGS="" - test "$TRAVIS_BRANCH" = master && TAGS="$TAGS latest" || true - test -n "$TRAVIS_TAG" && TAGS="$TAGS stable $TRAVIS_TAG" || true - echo "Tags are $TAGS" - -# This is in global context and runs for every stage that doesn't override it. -script: - - mkdir -p deploy - - source ./.compile_binaries - - ./striptests - - set -ex; build_"$BUILD"; set +x; - - ./.prepare_deploy - -# This is in global context and runs for every stage that doesn't override it. -after_failure: | - id - pwd - df -h - find . -name '*.log' -type f -exec grep "" /dev/null {} + - find . -ls - -# This is in global context and runs for every stage that doesn't override it. -deploy: - provider: gcs - skip_cleanup: true - access_key_id: GOOG7MDN7WEH6IIGBDCA - secret_access_key: - secure: Bcx2cT0/E2ikj7sdamVq52xlLZF9dz9ojGPtoKfPyQhkkZa+McVI4xgUSuyyoSxyKj77sofx2y8m6PJYYumT4g5hREV1tfeUkl0J2DQFMbGDYEt7kxVkXCxojNvhHwTzLFv0ezstrxWWxQm81BfQQ4U9lggRXtndAP4czZnOeHPINPSiue1QNwRAEw05r5UoIUJXy/5xyUrjIxn381pAs+gJqP2COeN9kTKYH53nS/AAws29RprfZFnPlo7xxWmcjRcdS5KPdGXI/c6tQp5zl2iTh510VC1PN2w1Wvnn/oNWhiNdqPyVDsojIX5+sS3nejzJA+KFMxXSBlyXIY3wPpS/MdscU79X6Q5f9ivsFfsm7gNBmxHUPNn0HAvU4ROT/CCE9j6jSbs5PC7QBo3CK4++jxAwE/pd9HUc2rs3k0ofx3rgveJ7txpy5yPKfwIIBi98kVKlC4w7dLvNTOfjW1Imt2yH87XTfsE0UIG9st1WII6s4l/WgBx2GuwKdt6+3QUYiAlCFckkxWi+fAvpHZUEL43Qxub5fN+ZV7Zib1n7opchH4QKGBb6/y0WaDCmtCfu0lppoe/TH6saOTjDFj67NJSElK6ZDxGZ3uw4R+ret2gm6WRKT2Oeub8J33VzSa7VkmFpMPrAAfPa9N1Z4ewBLoTmvxSg2A0dDrCdJio= - bucket: shellcheck-private - local_dir: deploy - on: - repo: koalaman/shellcheck - condition: $TRAVIS_BUILD_STAGE_NAME = Build - all_branches: true diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 2f8f79e..0000000 --- a/Dockerfile +++ /dev/null @@ -1,29 +0,0 @@ -# Build-only image -FROM ubuntu:18.04 AS build -USER root -WORKDIR /opt/shellCheck - -# Install OS deps -RUN apt-get update && apt-get install -y ghc cabal-install - -# Install Haskell deps -# (This is a separate copy/run so that source changes don't require rebuilding) -COPY ShellCheck.cabal ./ -RUN cabal update && cabal install --dependencies-only --ghc-options="-optlo-Os -split-sections" - -# Copy source and build it -COPY LICENSE shellcheck.hs ./ -COPY src src -RUN cabal build Paths_ShellCheck && \ - ghc -optl-static -optl-pthread -isrc -idist/build/autogen --make shellcheck -split-sections -optc-Wl,--gc-sections -optlo-Os && \ - strip --strip-all shellcheck - -RUN mkdir -p /out/bin && \ - cp shellcheck /out/bin/ - -# Resulting ShellCheck image -FROM scratch -LABEL maintainer="Vidar Holen " -WORKDIR /mnt -COPY --from=build /out / -ENTRYPOINT ["/bin/shellcheck"] diff --git a/build/README.md b/build/README.md new file mode 100644 index 0000000..eb745a0 --- /dev/null +++ b/build/README.md @@ -0,0 +1,13 @@ +This directory contains Dockerfiles for all builds. + +A build image will: + +* Run on Linux x86\_64 with vanilla Docker (no exceptions) +* Not contain any software that would restrict easy modification or copying +* Take a `cabal sdist` style tar.gz of the ShellCheck directory on stdin +* Output a tar.gz of artifacts on stdout, in a directory named for the arch + +This makes it simple to build any release without exotic hardware or software. + +An image can be built and tagged using `build_builder`, +and run on a source tarball using `run_builder`. diff --git a/build/build_builder b/build/build_builder new file mode 100755 index 0000000..b34b996 --- /dev/null +++ b/build/build_builder @@ -0,0 +1,12 @@ +#!/bin/sh +if [ $# -eq 0 ] +then + echo >&2 "No build image directories specified" + echo >&2 "Example: $0 build/*/" + exit 1 +fi + +for dir +do + ( cd "$dir" && docker build -t "$(cat tag)" . ) || exit 1 +done diff --git a/build/darwin.x86_64/Dockerfile b/build/darwin.x86_64/Dockerfile new file mode 100644 index 0000000..e7425fe --- /dev/null +++ b/build/darwin.x86_64/Dockerfile @@ -0,0 +1,31 @@ +# DIGEST:sha256:fa32af4677e2860a1c5950bc8c360f309e2a87e2ddfed27b642fddf7a6093b76 +FROM liushuyu/osxcross:latest + +ENV TARGET x86_64-apple-darwin18 +ENV TARGETNAME darwin.x86_64 + +# Build dependencies +USER root +ENV DEBIAN_FRONTEND noninteractive +RUN apt-get update && apt-get install -y ghc automake autoconf llvm curl + +# Build GHC +WORKDIR /ghc +RUN curl -L "https://downloads.haskell.org/~ghc/8.10.4/ghc-8.10.4-src.tar.xz" | tar xJ --strip-components=1 +RUN ./boot && ./configure --host x86_64-linux-gnu --build x86_64-linux-gnu --target "$TARGET" +RUN cp mk/flavours/quick-cross.mk mk/build.mk && make -j "$(nproc)" +RUN make install +RUN curl -L "https://downloads.haskell.org/~cabal/cabal-install-3.2.0.0/cabal-install-3.2.0.0-x86_64-unknown-linux.tar.xz" | tar xJv -C /usr/local/bin + +# Due to an apparent cabal bug, we specify our options directly to cabal +# It won't reuse caches if ghc-options are specified in ~/.cabal/config +ENV CABALOPTS "--with-ghc=$TARGET-ghc;--with-hc-pkg=$TARGET-ghc-pkg" + +# Prebuild the dependencies +RUN cabal update && IFS=';' && cabal install $CABALOPTS --lib Diff-0.4.0 base-compat-0.11.2 base-orphans-0.8.4 dlist-1.0 hashable-1.3.0.0 indexed-traversable-0.1.1 integer-logarithms-1.0.3.1 primitive-0.7.1.0 regex-base-0.94.0.0 splitmix-0.1.0.3 tagged-0.8.6.1 th-abstraction-0.4.2.0 transformers-compat-0.6.6 base-compat-batteries-0.11.2 time-compat-1.9.5 unordered-containers-0.2.13.0 data-fix-0.3.1 vector-0.12.2.0 scientific-0.3.6.2 regex-tdfa-1.3.1.0 random-1.2.0 distributive-0.6.2.1 attoparsec-0.13.2.5 uuid-types-1.0.3 comonad-5.0.8 bifunctors-5.5.10 assoc-1.0.2 these-1.1.1.1 strict-0.4.0.1 aeson-1.5.5.1 + +# Copy the build script +COPY build /usr/bin + +WORKDIR /scratch +ENTRYPOINT ["/usr/bin/build"] diff --git a/build/darwin.x86_64/build b/build/darwin.x86_64/build new file mode 100755 index 0000000..03dfde7 --- /dev/null +++ b/build/darwin.x86_64/build @@ -0,0 +1,14 @@ +#!/bin/sh +set -xe +{ + tar xzv --strip-components=1 + ./striptests + mkdir "$TARGETNAME" + cabal update + ( IFS=';'; cabal build $CABALOPTS ) + find . -name shellcheck -type f -exec mv {} "$TARGETNAME/" \; + ls -l "$TARGETNAME" + "$TARGET-strip" -Sx "$TARGETNAME/shellcheck" + ls -l "$TARGETNAME" +} >&2 +tar czv "$TARGETNAME" diff --git a/build/darwin.x86_64/tag b/build/darwin.x86_64/tag new file mode 100644 index 0000000..237a65c --- /dev/null +++ b/build/darwin.x86_64/tag @@ -0,0 +1 @@ +koalaman/scbuilder-darwin-x86_64 diff --git a/build/linux.aarch64/Dockerfile b/build/linux.aarch64/Dockerfile new file mode 100644 index 0000000..8bf8d6f --- /dev/null +++ b/build/linux.aarch64/Dockerfile @@ -0,0 +1,30 @@ +FROM ubuntu:20.04 + +ENV TARGET aarch64-linux-gnu +ENV TARGETNAME linux.aarch64 + +# Build dependencies +USER root +ENV DEBIAN_FRONTEND noninteractive +RUN apt-get update && apt-get install -y ghc automake autoconf build-essential llvm curl qemu-user-static gcc-$TARGET + +# Build GHC +WORKDIR /ghc +RUN curl -L "https://downloads.haskell.org/~ghc/8.10.4/ghc-8.10.4-src.tar.xz" | tar xJ --strip-components=1 +RUN ./boot && ./configure --host x86_64-linux-gnu --build x86_64-linux-gnu --target "$TARGET" +RUN cp mk/flavours/quick-cross.mk mk/build.mk && make -j "$(nproc)" +RUN make install +RUN curl -L "https://downloads.haskell.org/~cabal/cabal-install-3.2.0.0/cabal-install-3.2.0.0-x86_64-unknown-linux.tar.xz" | tar xJv -C /usr/local/bin + +# Due to an apparent cabal bug, we specify our options directly to cabal +# It won't reuse caches if ghc-options are specified in ~/.cabal/config +ENV CABALOPTS "--ghc-options;-split-sections -optc-Os -optc-Wl,--gc-sections;--with-ghc=$TARGET-ghc;--with-hc-pkg=$TARGET-ghc-pkg" + +# Prebuild the dependencies +RUN cabal update && IFS=';' && cabal install $CABALOPTS --lib Diff-0.4.0 base-compat-0.11.2 base-orphans-0.8.4 dlist-1.0 hashable-1.3.0.0 indexed-traversable-0.1.1 integer-logarithms-1.0.3.1 primitive-0.7.1.0 regex-base-0.94.0.0 splitmix-0.1.0.3 tagged-0.8.6.1 th-abstraction-0.4.2.0 transformers-compat-0.6.6 base-compat-batteries-0.11.2 time-compat-1.9.5 unordered-containers-0.2.13.0 data-fix-0.3.1 vector-0.12.2.0 scientific-0.3.6.2 regex-tdfa-1.3.1.0 random-1.2.0 distributive-0.6.2.1 attoparsec-0.13.2.5 uuid-types-1.0.3 comonad-5.0.8 bifunctors-5.5.10 assoc-1.0.2 these-1.1.1.1 strict-0.4.0.1 aeson-1.5.5.1 + +# Copy the build script +COPY build /usr/bin + +WORKDIR /scratch +ENTRYPOINT ["/usr/bin/build"] diff --git a/build/linux.aarch64/build b/build/linux.aarch64/build new file mode 100755 index 0000000..1164dc1 --- /dev/null +++ b/build/linux.aarch64/build @@ -0,0 +1,15 @@ +#!/bin/sh +set -xe +{ + tar xzv --strip-components=1 + ./striptests + mkdir "$TARGETNAME" + cabal update + ( IFS=';'; cabal build $CABALOPTS --enable-executable-static ) + find . -name shellcheck -type f -exec mv {} "$TARGETNAME/" \; + ls -l "$TARGETNAME" + "$TARGET-strip" -s "$TARGETNAME/shellcheck" + ls -l "$TARGETNAME" + qemu-aarch64-static "$TARGETNAME/shellcheck" --version +} >&2 +tar czv "$TARGETNAME" diff --git a/build/linux.aarch64/tag b/build/linux.aarch64/tag new file mode 100644 index 0000000..6788e14 --- /dev/null +++ b/build/linux.aarch64/tag @@ -0,0 +1 @@ +koalaman/scbuilder-linux-aarch64 diff --git a/build/linux.armv6hf/Dockerfile b/build/linux.armv6hf/Dockerfile new file mode 100644 index 0000000..fee6d4c --- /dev/null +++ b/build/linux.armv6hf/Dockerfile @@ -0,0 +1,59 @@ +# I've again spent days trying to get a working armv6hf compiler going. +# God only knows how many recompilations of GCC, GHC, libraries, and +# ShellCheck itself, has gone into it. +# +# I tried Debian's toolchain. I tried my custom one built according to +# RPi `gcc -v`. I tried GHC9, glibc, musl, registerised vs not, but +# nothing has yielded an armv6hf binary that does not immediately +# segfault on qemu-arm-static or the RPi itself. +# +# I then tried the same but with armv7hf. Same story. +# +# Emulating the entire userspace with balenalib again? Very strange build +# failures where programs would fail to execute with > ~100 arguments. +# +# Finally, creating our own appears to work when using a custom QEmu +# patched to follow execve calls. +# +# PS: $100 bounty for getting a RPi1 compatible static build going +# with cross-compilation, similar to what the aarch64 build does. +# + +FROM ubuntu:20.04 + +ENV TARGETNAME linux.armv6hf + +# Build QEmu with execve follow support +USER root +ENV DEBIAN_FRONTEND noninteractive +RUN apt-get update +RUN apt-get install -y build-essential git ninja-build python3 pkg-config libglib2.0-dev libpixman-1-dev +WORKDIR /build +RUN git clone --depth 1 https://github.com/koalaman/qemu +RUN cd qemu && ./configure --static && cd build && ninja qemu-arm +RUN cp qemu/build/qemu-arm /build/qemu-arm-static +ENV QEMU_EXECVE 1 + +# Set up an armv6 userspace +WORKDIR / +RUN apt-get install -y debootstrap qemu-user-static +# We expect this to fail if the host doesn't have binfmt qemu support +RUN qemu-debootstrap --arch armhf bullseye pi http://mirrordirector.raspbian.org/raspbian || [ -e /pi/etc/issue ] +RUN cp /build/qemu-arm-static /pi/usr/bin/qemu-arm-static +RUN printf > /bin/pirun '%s\n' '#!/bin/sh' 'chroot /pi /usr/bin/qemu-arm-static /usr/bin/env "$@"' && chmod +x /bin/pirun +# If the debootstrap process didn't finish, continue it +RUN [ ! -e /pi/debootstrap ] || pirun '/debootstrap/debootstrap' --second-stage + +# Install deps in the chroot +RUN pirun apt-get update +RUN pirun apt-get install -y ghc cabal-install + +# Finally we can build the current dependencies. This takes hours. +ENV CABALOPTS "--ghc-options;-split-sections -optc-Os -optc-Wl,--gc-sections;--gcc-options;-Os -Wl,--gc-sections -ffunction-sections -fdata-sections" +RUN pirun cabal update +RUN IFS=";" && pirun cabal install --lib $CABALOPTS Diff-0.4.0 base-compat-0.11.2 base-orphans-0.8.4 dlist-1.0 hashable-1.3.1.0 indexed-traversable-0.1.1 integer-logarithms-1.0.3.1 primitive-0.7.1.0 regex-base-0.94.0.1 splitmix-0.1.0.3 tagged-0.8.6.1 th-abstraction-0.4.2.0 transformers-compat-0.6.6 base-compat-batteries-0.11.2 time-compat-1.9.5 unordered-containers-0.2.13.0 data-fix-0.3.1 vector-0.12.2.0 scientific-0.3.6.2 regex-tdfa-1.3.1.0 random-1.2.0 distributive-0.6.2.1 attoparsec-0.13.2.5 uuid-types-1.0.4 comonad-5.0.8 bifunctors-5.5.10 assoc-1.0.2 these-1.1.1.1 strict-0.4.0.1 aeson-1.5.6.0 + +# Copy the build script +WORKDIR /pi/scratch +COPY build /pi/usr/bin +ENTRYPOINT ["/bin/pirun", "/usr/bin/build"] diff --git a/build/linux.armv6hf/build b/build/linux.armv6hf/build new file mode 100755 index 0000000..4f2c7bc --- /dev/null +++ b/build/linux.armv6hf/build @@ -0,0 +1,16 @@ +#!/bin/sh +set -xe +cd /scratch +{ + tar xzv --strip-components=1 + ./striptests + mkdir "$TARGETNAME" + # This script does not cabal update because compiling anything new is slow + ( IFS=';'; cabal build $CABALOPTS --enable-executable-static ) + find . -name shellcheck -type f -exec mv {} "$TARGETNAME/" \; + ls -l "$TARGETNAME" + strip -s "$TARGETNAME/shellcheck" + ls -l "$TARGETNAME" + "$TARGETNAME/shellcheck" --version +} >&2 +tar czv "$TARGETNAME" diff --git a/build/linux.armv6hf/tag b/build/linux.armv6hf/tag new file mode 100644 index 0000000..9172c5c --- /dev/null +++ b/build/linux.armv6hf/tag @@ -0,0 +1 @@ +koalaman/scbuilder-linux-armv6hf diff --git a/build/linux.x86_64/Dockerfile b/build/linux.x86_64/Dockerfile new file mode 100644 index 0000000..f0ad16a --- /dev/null +++ b/build/linux.x86_64/Dockerfile @@ -0,0 +1,26 @@ +FROM ubuntu:20.04 + +ENV TARGETNAME linux.x86_64 + +# Install GHC and cabal +USER root +ENV DEBIAN_FRONTEND noninteractive +RUN apt-get update && apt-get install -y ghc curl xz-utils + +# So we'd like a later version of Cabal that supports --enable-executable-static, +# but we can't use Ubuntu 20.10 because coreutils has switched to new syscalls that +# the TravisCI kernel doesn't support. Download it manually. +RUN curl "https://downloads.haskell.org/~cabal/cabal-install-3.2.0.0/cabal-install-3.2.0.0-x86_64-unknown-linux.tar.xz" | tar xJv -C /usr/bin + +# Use ld.bfd instead of ld.gold due to +# x86_64-linux-gnu/libpthread.a(pthread_cond_init.o)(.note.stapsdt+0x14): error: +# relocation refers to local symbol "" [2], which is defined in a discarded section +ENV CABALOPTS "--ghc-options;-optl-Wl,-fuse-ld=bfd -split-sections -optc-Os -optc-Wl,--gc-sections" + +# Other archs pre-build dependencies here, but this one doesn't to detect ecosystem movement + +# Copy the build script +COPY build /usr/bin + +WORKDIR /scratch +ENTRYPOINT ["/usr/bin/build"] diff --git a/build/linux.x86_64/build b/build/linux.x86_64/build new file mode 100755 index 0000000..ba598e6 --- /dev/null +++ b/build/linux.x86_64/build @@ -0,0 +1,15 @@ +#!/bin/sh +set -xe +{ + tar xzv --strip-components=1 + ./striptests + mkdir "$TARGETNAME" + cabal update + ( IFS=';'; cabal build $CABALOPTS --enable-executable-static ) + find . -name shellcheck -type f -exec mv {} "$TARGETNAME/" \; + ls -l "$TARGETNAME" + strip -s "$TARGETNAME/shellcheck" + ls -l "$TARGETNAME" + "$TARGETNAME/shellcheck" --version +} >&2 +tar czv "$TARGETNAME" diff --git a/build/linux.x86_64/tag b/build/linux.x86_64/tag new file mode 100644 index 0000000..f0224de --- /dev/null +++ b/build/linux.x86_64/tag @@ -0,0 +1 @@ +koalaman/scbuilder-linux-x86_64 diff --git a/build/run_builder b/build/run_builder new file mode 100755 index 0000000..d6de27b --- /dev/null +++ b/build/run_builder @@ -0,0 +1,30 @@ +#!/bin/bash +if [ $# -lt 2 ] +then + echo >&2 "This script builds a source archive (as produced by cabal sdist)" + echo >&2 "Usage: $0 sourcefile.tar.gz builddir..." + exit 1 +fi + +file=$(realpath "$1") +shift + +if [ ! -e "$file" ] +then + echo >&2 "$file does not exist" + exit 1 +fi + +set -ex -o pipefail + +for dir +do + tagfile="$dir/tag" + if [ ! -e "$tagfile" ] + then + echo >&2 "$tagfile does not exist" + exit 2 + fi + + docker run -i "$(< "$tagfile")" < "$file" | tar xz +done diff --git a/build/windows.x86_64/Dockerfile b/build/windows.x86_64/Dockerfile new file mode 100644 index 0000000..3eb20fa --- /dev/null +++ b/build/windows.x86_64/Dockerfile @@ -0,0 +1,27 @@ +FROM ubuntu:20.04 + +ENV TARGETNAME windows.x86_64 + +# We don't need wine32, even though it complains +USER root +ENV DEBIAN_FRONTEND noninteractive +RUN apt-get update && apt-get install -y curl busybox wine + +# Fetch Windows version, will be available under z:\haskell +WORKDIR /haskell +RUN curl -L "https://downloads.haskell.org/~ghc/8.10.4/ghc-8.10.4-x86_64-unknown-mingw32.tar.xz" | tar xJ --strip-components=1 +WORKDIR /haskell/bin +RUN curl -L "https://downloads.haskell.org/~cabal/cabal-install-3.2.0.0/cabal-install-3.2.0.0-x86_64-unknown-mingw32.zip" | busybox unzip - +RUN curl -L "https://curl.se/windows/dl-7.75.0/curl-7.75.0-win64-mingw.zip" | busybox unzip - && mv curl-7.75.0-win64-mingw/bin/* . +ENV WINEPATH /haskell/bin + +# It's unknown whether Cabal on Windows suffers from the same issue +# that necessitated this but I don't care enough to find out +ENV CABALOPTS "--ghc-options;-split-sections -optc-Os -optc-Wl,--gc-sections" + +# Precompile some deps to speed up later builds. This list is just copied from `cabal build` +RUN wine /haskell/bin/cabal.exe update && IFS=';' && wine /haskell/bin/cabal.exe install $CABALOPTS --lib Diff-0.4.0 base-compat-0.11.2 base-orphans-0.8.4 dlist-1.0 hashable-1.3.0.0 indexed-traversable-0.1.1 integer-logarithms-1.0.3.1 primitive-0.7.1.0 regex-base-0.94.0.0 splitmix-0.1.0.3 tagged-0.8.6.1 th-abstraction-0.4.2.0 transformers-compat-0.6.6 base-compat-batteries-0.11.2 time-compat-1.9.5 unordered-containers-0.2.13.0 data-fix-0.3.1 vector-0.12.2.0 scientific-0.3.6.2 regex-tdfa-1.3.1.0 random-1.2.0 distributive-0.6.2.1 attoparsec-0.13.2.5 uuid-types-1.0.3 comonad-5.0.8 bifunctors-5.5.10 assoc-1.0.2 these-1.1.1.1 strict-0.4.0.1 aeson-1.5.5.1 + +COPY build /usr/bin +WORKDIR /scratch +ENTRYPOINT ["/usr/bin/build"] diff --git a/build/windows.x86_64/build b/build/windows.x86_64/build new file mode 100755 index 0000000..bedb870 --- /dev/null +++ b/build/windows.x86_64/build @@ -0,0 +1,19 @@ +#!/bin/sh +cabal() { + wine /haskell/bin/cabal.exe "$@" +} + +set -xe +{ + tar xzv --strip-components=1 + ./striptests + mkdir "$TARGETNAME" + cabal update + ( IFS=';'; cabal build $CABALOPTS ) + find dist*/ -name shellcheck.exe -type f -ls -exec mv {} "$TARGETNAME/" \; + ls -l "$TARGETNAME" + wine "/haskell/mingw/bin/strip.exe" -s "$TARGETNAME/shellcheck.exe" + ls -l "$TARGETNAME" + wine "$TARGETNAME/shellcheck.exe" --version +} >&2 +tar czv "$TARGETNAME" diff --git a/build/windows.x86_64/tag b/build/windows.x86_64/tag new file mode 100644 index 0000000..a85921b --- /dev/null +++ b/build/windows.x86_64/tag @@ -0,0 +1 @@ +koalaman/scbuilder-windows-x86_64 From 8442695b736dd10a3cf8e8be3b55986a29082e5a Mon Sep 17 00:00:00 2001 From: Viacheslav Vasilyev Date: Sat, 6 Mar 2021 14:19:08 +0200 Subject: [PATCH 129/459] suppress ntlm error messages in Windows build when building for windows there are many error message like below ``` 003a:err:winediag:SECUR32_initNTLMSP ntlm_auth was not found or is outdated. Make sure that ntlm_auth >= 3.0.25 is in your path. Usually, you can find it in the winbind package of your distribution. ``` --- build/windows.x86_64/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/windows.x86_64/Dockerfile b/build/windows.x86_64/Dockerfile index 3eb20fa..c5fa07b 100644 --- a/build/windows.x86_64/Dockerfile +++ b/build/windows.x86_64/Dockerfile @@ -5,7 +5,7 @@ ENV TARGETNAME windows.x86_64 # We don't need wine32, even though it complains USER root ENV DEBIAN_FRONTEND noninteractive -RUN apt-get update && apt-get install -y curl busybox wine +RUN apt-get update && apt-get install -y curl busybox wine winbind # Fetch Windows version, will be available under z:\haskell WORKDIR /haskell From 88cd21fd0f679d96b44abbbb343f1d9eb66b2c28 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 22 Feb 2021 21:28:14 -0800 Subject: [PATCH 130/459] Fix missing +x with new cabal and use previous release deps for caching --- .github/workflows/build.yml | 4 ++-- build/darwin.x86_64/Dockerfile | 2 +- build/darwin.x86_64/build | 2 +- build/linux.aarch64/Dockerfile | 2 +- build/linux.aarch64/build | 2 +- build/linux.armv6hf/Dockerfile | 2 +- build/linux.armv6hf/build | 2 +- build/linux.x86_64/build | 2 +- build/windows.x86_64/Dockerfile | 4 ++-- build/windows.x86_64/build | 2 +- 10 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cd9fca0..707b9a2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,4 +1,4 @@ -name: Build Lol +name: Build ShellCheck # Run this workflow every time a new commit pushed to your repository on: push @@ -21,7 +21,7 @@ jobs: run: | mkdir source cabal sdist - mv dist/*.tar.gz source/source.tar.gz + mv dist-newstyle/sdist/*.tar.gz source/source.tar.gz - name: Deduce tags run: | diff --git a/build/darwin.x86_64/Dockerfile b/build/darwin.x86_64/Dockerfile index e7425fe..ecd1cad 100644 --- a/build/darwin.x86_64/Dockerfile +++ b/build/darwin.x86_64/Dockerfile @@ -22,7 +22,7 @@ RUN curl -L "https://downloads.haskell.org/~cabal/cabal-install-3.2.0.0/cabal-in ENV CABALOPTS "--with-ghc=$TARGET-ghc;--with-hc-pkg=$TARGET-ghc-pkg" # Prebuild the dependencies -RUN cabal update && IFS=';' && cabal install $CABALOPTS --lib Diff-0.4.0 base-compat-0.11.2 base-orphans-0.8.4 dlist-1.0 hashable-1.3.0.0 indexed-traversable-0.1.1 integer-logarithms-1.0.3.1 primitive-0.7.1.0 regex-base-0.94.0.0 splitmix-0.1.0.3 tagged-0.8.6.1 th-abstraction-0.4.2.0 transformers-compat-0.6.6 base-compat-batteries-0.11.2 time-compat-1.9.5 unordered-containers-0.2.13.0 data-fix-0.3.1 vector-0.12.2.0 scientific-0.3.6.2 regex-tdfa-1.3.1.0 random-1.2.0 distributive-0.6.2.1 attoparsec-0.13.2.5 uuid-types-1.0.3 comonad-5.0.8 bifunctors-5.5.10 assoc-1.0.2 these-1.1.1.1 strict-0.4.0.1 aeson-1.5.5.1 +RUN cabal update && IFS=';' && cabal install --dependencies-only $CABALOPTS ShellCheck # Copy the build script COPY build /usr/bin diff --git a/build/darwin.x86_64/build b/build/darwin.x86_64/build index 03dfde7..53857e8 100755 --- a/build/darwin.x86_64/build +++ b/build/darwin.x86_64/build @@ -2,7 +2,7 @@ set -xe { tar xzv --strip-components=1 - ./striptests + chmod +x striptests && ./striptests mkdir "$TARGETNAME" cabal update ( IFS=';'; cabal build $CABALOPTS ) diff --git a/build/linux.aarch64/Dockerfile b/build/linux.aarch64/Dockerfile index 8bf8d6f..60537b3 100644 --- a/build/linux.aarch64/Dockerfile +++ b/build/linux.aarch64/Dockerfile @@ -21,7 +21,7 @@ RUN curl -L "https://downloads.haskell.org/~cabal/cabal-install-3.2.0.0/cabal-in ENV CABALOPTS "--ghc-options;-split-sections -optc-Os -optc-Wl,--gc-sections;--with-ghc=$TARGET-ghc;--with-hc-pkg=$TARGET-ghc-pkg" # Prebuild the dependencies -RUN cabal update && IFS=';' && cabal install $CABALOPTS --lib Diff-0.4.0 base-compat-0.11.2 base-orphans-0.8.4 dlist-1.0 hashable-1.3.0.0 indexed-traversable-0.1.1 integer-logarithms-1.0.3.1 primitive-0.7.1.0 regex-base-0.94.0.0 splitmix-0.1.0.3 tagged-0.8.6.1 th-abstraction-0.4.2.0 transformers-compat-0.6.6 base-compat-batteries-0.11.2 time-compat-1.9.5 unordered-containers-0.2.13.0 data-fix-0.3.1 vector-0.12.2.0 scientific-0.3.6.2 regex-tdfa-1.3.1.0 random-1.2.0 distributive-0.6.2.1 attoparsec-0.13.2.5 uuid-types-1.0.3 comonad-5.0.8 bifunctors-5.5.10 assoc-1.0.2 these-1.1.1.1 strict-0.4.0.1 aeson-1.5.5.1 +RUN cabal update && IFS=';' && cabal install --dependencies-only $CABALOPTS ShellCheck # Copy the build script COPY build /usr/bin diff --git a/build/linux.aarch64/build b/build/linux.aarch64/build index 1164dc1..f8001aa 100755 --- a/build/linux.aarch64/build +++ b/build/linux.aarch64/build @@ -2,7 +2,7 @@ set -xe { tar xzv --strip-components=1 - ./striptests + chmod +x striptests && ./striptests mkdir "$TARGETNAME" cabal update ( IFS=';'; cabal build $CABALOPTS --enable-executable-static ) diff --git a/build/linux.armv6hf/Dockerfile b/build/linux.armv6hf/Dockerfile index fee6d4c..bd5795c 100644 --- a/build/linux.armv6hf/Dockerfile +++ b/build/linux.armv6hf/Dockerfile @@ -51,7 +51,7 @@ RUN pirun apt-get install -y ghc cabal-install # Finally we can build the current dependencies. This takes hours. ENV CABALOPTS "--ghc-options;-split-sections -optc-Os -optc-Wl,--gc-sections;--gcc-options;-Os -Wl,--gc-sections -ffunction-sections -fdata-sections" RUN pirun cabal update -RUN IFS=";" && pirun cabal install --lib $CABALOPTS Diff-0.4.0 base-compat-0.11.2 base-orphans-0.8.4 dlist-1.0 hashable-1.3.1.0 indexed-traversable-0.1.1 integer-logarithms-1.0.3.1 primitive-0.7.1.0 regex-base-0.94.0.1 splitmix-0.1.0.3 tagged-0.8.6.1 th-abstraction-0.4.2.0 transformers-compat-0.6.6 base-compat-batteries-0.11.2 time-compat-1.9.5 unordered-containers-0.2.13.0 data-fix-0.3.1 vector-0.12.2.0 scientific-0.3.6.2 regex-tdfa-1.3.1.0 random-1.2.0 distributive-0.6.2.1 attoparsec-0.13.2.5 uuid-types-1.0.4 comonad-5.0.8 bifunctors-5.5.10 assoc-1.0.2 these-1.1.1.1 strict-0.4.0.1 aeson-1.5.6.0 +RUN IFS=";" && pirun cabal install --dependencies-only $CABALOPTS ShellCheck # Copy the build script WORKDIR /pi/scratch diff --git a/build/linux.armv6hf/build b/build/linux.armv6hf/build index 4f2c7bc..daa94d9 100755 --- a/build/linux.armv6hf/build +++ b/build/linux.armv6hf/build @@ -3,7 +3,7 @@ set -xe cd /scratch { tar xzv --strip-components=1 - ./striptests + chmod +x striptests && ./striptests mkdir "$TARGETNAME" # This script does not cabal update because compiling anything new is slow ( IFS=';'; cabal build $CABALOPTS --enable-executable-static ) diff --git a/build/linux.x86_64/build b/build/linux.x86_64/build index ba598e6..099f127 100755 --- a/build/linux.x86_64/build +++ b/build/linux.x86_64/build @@ -2,7 +2,7 @@ set -xe { tar xzv --strip-components=1 - ./striptests + chmod +x striptests && ./striptests mkdir "$TARGETNAME" cabal update ( IFS=';'; cabal build $CABALOPTS --enable-executable-static ) diff --git a/build/windows.x86_64/Dockerfile b/build/windows.x86_64/Dockerfile index 3eb20fa..b412c1c 100644 --- a/build/windows.x86_64/Dockerfile +++ b/build/windows.x86_64/Dockerfile @@ -19,8 +19,8 @@ ENV WINEPATH /haskell/bin # that necessitated this but I don't care enough to find out ENV CABALOPTS "--ghc-options;-split-sections -optc-Os -optc-Wl,--gc-sections" -# Precompile some deps to speed up later builds. This list is just copied from `cabal build` -RUN wine /haskell/bin/cabal.exe update && IFS=';' && wine /haskell/bin/cabal.exe install $CABALOPTS --lib Diff-0.4.0 base-compat-0.11.2 base-orphans-0.8.4 dlist-1.0 hashable-1.3.0.0 indexed-traversable-0.1.1 integer-logarithms-1.0.3.1 primitive-0.7.1.0 regex-base-0.94.0.0 splitmix-0.1.0.3 tagged-0.8.6.1 th-abstraction-0.4.2.0 transformers-compat-0.6.6 base-compat-batteries-0.11.2 time-compat-1.9.5 unordered-containers-0.2.13.0 data-fix-0.3.1 vector-0.12.2.0 scientific-0.3.6.2 regex-tdfa-1.3.1.0 random-1.2.0 distributive-0.6.2.1 attoparsec-0.13.2.5 uuid-types-1.0.3 comonad-5.0.8 bifunctors-5.5.10 assoc-1.0.2 these-1.1.1.1 strict-0.4.0.1 aeson-1.5.5.1 +# Precompile some deps to speed up later builds +RUN wine /haskell/bin/cabal.exe update && IFS=';' && wine /haskell/bin/cabal.exe install --lib --dependencies-only $CABALOPTS ShellCheck COPY build /usr/bin WORKDIR /scratch diff --git a/build/windows.x86_64/build b/build/windows.x86_64/build index bedb870..7bf186e 100755 --- a/build/windows.x86_64/build +++ b/build/windows.x86_64/build @@ -6,7 +6,7 @@ cabal() { set -xe { tar xzv --strip-components=1 - ./striptests + chmod +x striptests && ./striptests mkdir "$TARGETNAME" cabal update ( IFS=';'; cabal build $CABALOPTS ) From f02c297fddb9de8b45a5fedb912fb8c1f96b79f6 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 11 Mar 2021 23:04:17 -0800 Subject: [PATCH 131/459] Merge parser and analyzer shebang parsing --- src/ShellCheck/ASTLib.hs | 44 +++++++++++++++++++++++++++++++++++ src/ShellCheck/Analytics.hs | 2 +- src/ShellCheck/AnalyzerLib.hs | 15 ++---------- src/ShellCheck/Parser.hs | 18 +++----------- test/shellcheck.hs | 2 ++ 5 files changed, 52 insertions(+), 29 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index c5f1735..019fc3c 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -17,9 +17,11 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . -} +{-# LANGUAGE TemplateHaskell #-} module ShellCheck.ASTLib where import ShellCheck.AST +import ShellCheck.Regex import Control.Monad.Writer import Control.Monad @@ -31,6 +33,8 @@ import Data.Maybe import qualified Data.Map as Map import Numeric (showHex) +import Test.QuickCheck + arguments (T_SimpleCommand _ _ (cmd:args)) = args -- Is this a type of loop? @@ -672,3 +676,43 @@ isAnnotationIgnoringCode code t = where hasNum (DisableComment from to) = code >= from && code < to hasNum _ = False + +prop_executableFromShebang1 = executableFromShebang "/bin/sh" == "sh" +prop_executableFromShebang2 = executableFromShebang "/bin/bash" == "bash" +prop_executableFromShebang3 = executableFromShebang "/usr/bin/env ksh" == "ksh" +prop_executableFromShebang4 = executableFromShebang "/usr/bin/env -S foo=bar bash -x" == "bash" +prop_executableFromShebang5 = executableFromShebang "/usr/bin/env --split-string=bash -x" == "bash" +prop_executableFromShebang6 = executableFromShebang "/usr/bin/env --split-string=foo=bar bash -x" == "bash" +prop_executableFromShebang7 = executableFromShebang "/usr/bin/env --split-string bash -x" == "bash" +prop_executableFromShebang8 = executableFromShebang "/usr/bin/env --split-string foo=bar bash -x" == "bash" +prop_executableFromShebang9 = executableFromShebang "/usr/bin/env foo=bar dash" == "dash" +prop_executableFromShebang10 = executableFromShebang "/bin/busybox sh" == "ash" +prop_executableFromShebang11 = executableFromShebang "/bin/busybox ash" == "ash" + +-- Get the shell executable from a string like '/usr/bin/env bash' +executableFromShebang :: String -> String +executableFromShebang = shellFor + where + re = mkRegex "/env +(-S|--split-string=?)? *(.*)" + shellFor s | s `matches` re = + case matchRegex re s of + Just [flag, shell] -> fromEnvArgs (words shell) + _ -> "" + shellFor sb = + case words sb of + [] -> "" + [x] -> basename x + (first:second:args) | basename first == "busybox" -> + case basename second of + "sh" -> "ash" -- busybox sh is ash + x -> x + (first:args) | basename first == "env" -> + fromEnvArgs args + (first:_) -> basename first + + fromEnvArgs args = fromMaybe "" $ find (notElem '=') $ skipFlags args + basename s = reverse . takeWhile (/= '/') . reverse $ s + skipFlags = dropWhile ("-" `isPrefixOf`) + +return [] +runTests = $quickCheckAll diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 160ae95..9035e04 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -591,7 +591,7 @@ prop_checkShebang10= verifyNotTree checkShebang "#!foo\n# shellcheck shell=sh ig prop_checkShebang11= verifyTree checkShebang "#!/bin/sh/\ntrue" prop_checkShebang12= verifyTree checkShebang "#!/bin/sh/ -xe\ntrue" prop_checkShebang13= verifyTree checkShebang "#!/bin/busybox sh" -prop_checkShebang14= verifyTree checkShebang "#!/bin/busybox sh\n# shellcheck shell=sh\n" +prop_checkShebang14= verifyNotTree checkShebang "#!/bin/busybox sh\n# shellcheck shell=sh\n" prop_checkShebang15= verifyNotTree checkShebang "#!/bin/busybox sh\n# shellcheck shell=dash\n" prop_checkShebang16= verifyTree checkShebang "#!/bin/busybox ash" prop_checkShebang17= verifyNotTree checkShebang "#!/bin/busybox ash\n# shellcheck shell=dash\n" diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index d3b1134..5b97e87 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -240,6 +240,8 @@ prop_determineShell7 = determineShellTest "#! /bin/ash" == Dash prop_determineShell8 = determineShellTest' (Just Ksh) "#!/bin/sh" == Sh prop_determineShell9 = determineShellTest "#!/bin/env -S dash -x" == Dash prop_determineShell10 = determineShellTest "#!/bin/env --split-string= dash -x" == Dash +prop_determineShell11 = determineShellTest "#!/bin/busybox sh" == Dash -- busybox sh is a specific shell, not posix sh +prop_determineShell12 = determineShellTest "#!/bin/busybox ash" == Dash determineShellTest = determineShellTest' Nothing determineShellTest' fallbackShell = determineShell fallbackShell . fromJust . prRoot . pScript @@ -253,19 +255,6 @@ determineShell fallbackShell t = fromMaybe Bash $ headOrDefault (fromShebang s) [s | ShellOverride s <- annotations] fromShebang (T_Script _ (T_Literal _ s) _) = executableFromShebang s --- Given a string like "/bin/bash" or "/usr/bin/env dash", --- return the shell basename like "bash" or "dash" -executableFromShebang :: String -> String -executableFromShebang = shellFor - where - shellFor s | "/env " `isInfixOf` s = case matchRegex re s of - Just [flag, shell] -> shell - _ -> "" - shellFor s | ' ' `elem` s = shellFor $ takeWhile (/= ' ') s - shellFor s = reverse . takeWhile (/= '/') . reverse $ s - re = mkRegex "/env +(-S|--split-string=?)? *([^ ]*)" - - -- Given a root node, make a map from Id to parent Token. -- This is used to populate parentMap in Parameters getParentTree :: Token -> Map.Map Id Token diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 70ea05d..f32c20b 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -24,7 +24,7 @@ module ShellCheck.Parser (parseScript, runTests) where import ShellCheck.AST -import ShellCheck.ASTLib +import ShellCheck.ASTLib hiding (runTests) import ShellCheck.Data import ShellCheck.Interface @@ -3216,8 +3216,8 @@ readScriptFile sourced = do let ignoreShebang = shellAnnotationSpecified || shellFlagSpecified unless ignoreShebang $ - verifyShebang pos (getShell shebangString) - if ignoreShebang || isValidShell (getShell shebangString) /= Just False + verifyShebang pos (executableFromShebang shebangString) + if ignoreShebang || isValidShell (executableFromShebang shebangString) /= Just False then do commands <- withAnnotations annotations readCompoundListOrEmpty id <- endSpan start @@ -3231,18 +3231,6 @@ readScriptFile sourced = do return $ T_Script id shebang [] where - basename s = reverse . takeWhile (/= '/') . reverse $ s - skipFlags = dropWhile ("-" `isPrefixOf`) - getShell sb = - case words sb of - [] -> "" - [x] -> basename x - (first:args) | basename first == "env" -> - fromMaybe "" $ find (notElem '=') $ skipFlags args - (first:second:args) | basename first == "busybox" -> - second - (first:_) -> basename first - verifyShebang pos s = do case isValidShell s of Just True -> return () diff --git a/test/shellcheck.hs b/test/shellcheck.hs index ac84116..e463403 100644 --- a/test/shellcheck.hs +++ b/test/shellcheck.hs @@ -4,6 +4,7 @@ import Control.Monad import System.Exit import qualified ShellCheck.Analytics import qualified ShellCheck.AnalyzerLib +import qualified ShellCheck.ASTLib import qualified ShellCheck.Checker import qualified ShellCheck.Checks.Commands import qualified ShellCheck.Checks.Custom @@ -17,6 +18,7 @@ main = do results <- sequence [ ShellCheck.Analytics.runTests ,ShellCheck.AnalyzerLib.runTests + ,ShellCheck.ASTLib.runTests ,ShellCheck.Checker.runTests ,ShellCheck.Checks.Commands.runTests ,ShellCheck.Checks.Custom.runTests From 98952df35b834315c50c2ce6fe8f96516bf2a7ae Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 20 Mar 2021 13:58:37 -0700 Subject: [PATCH 132/459] Improve warnings on backslashes in comments --- CHANGELOG.md | 1 + src/ShellCheck/Parser.hs | 24 +++++++++++++++++------- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d7a1ac..9eec957 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## Git ### Added - `disable` directives can now be a range, e.g. `disable=SC3000-SC4000` +- SC1143: Warn about line continuations in comments - SC2259/SC2260: Warn when redirections override pipes - SC2261: Warn about multiple competing redirections - SC2262/SC2263: Warn about aliases declared and used in the same parsing unit diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 8db1991..3c16d5d 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -87,11 +87,23 @@ extglobStart = oneOf extglobStartChars unicodeDoubleQuotes = "\x201C\x201D\x2033\x2036" unicodeSingleQuotes = "\x2018\x2019" -prop_spacing = isOk spacing " \\\n # Comment" +prop_spacing1 = isOk spacing " \\\n # Comment" +prop_spacing2 = isOk spacing "# We can continue lines with \\" +prop_spacing3 = isWarning spacing " \\\n # --verbose=true \\" spacing = do - x <- many (many1 linewhitespace <|> try (string "\\\n" >> return "")) + x <- many (many1 linewhitespace <|> continuation) optional readComment return $ concat x + where + continuation = do + try (string "\\\n") + -- The line was continued. Warn if this next line is a comment with a trailing \ + whitespace <- many linewhitespace + optional $ do + x <- readComment + when ("\\" `isSuffixOf` x) $ + parseProblem ErrorC 1143 "This backslash is part of a comment and does not continue the line." + return whitespace spacing1 = do spacing <- spacing @@ -1040,13 +1052,9 @@ readComment = do readAnyComment prop_readAnyComment = isOk readAnyComment "# Comment" -prop_readAnyComment1 = not $ isOk readAnyComment "# Comment \\\n" readAnyComment = do char '#' - comment <- many $ noneOf "\\\r\n" - bs <- many $ oneOf "\\" - unless (null bs) (fail "Backslash in or directly after comment") - return comment + many $ noneOf "\r\n" prop_readNormalWord = isOk readNormalWord "'foo'\"bar\"{1..3}baz$(lol)" prop_readNormalWord2 = isOk readNormalWord "foo**(foo)!!!(@@(bar))" @@ -1409,6 +1417,8 @@ readNormalEscaped = called "escaped char" $ do do next <- quotable <|> oneOf "?*@!+[]{}.,~#" when (next == ' ') $ checkTrailingSpaces pos <|> return () + -- Check if this line is followed by a commented line with a trailing backslash + when (next == '\n') $ try . lookAhead $ void spacing return $ if next == '\n' then "" else [next] <|> do From 087865c68019597fc92e20f7e3276457018986d3 Mon Sep 17 00:00:00 2001 From: Matthias Diener Date: Sat, 20 Mar 2021 20:43:18 -0500 Subject: [PATCH 133/459] Clarify 'which' --- src/ShellCheck/Checks/Commands.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index b3e81a1..d7452e2 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -1056,7 +1056,7 @@ checkFindRedirections = CommandCheck (Basename "find") f prop_checkWhich = verify checkWhich "which '.+'" checkWhich = CommandCheck (Basename "which") $ - \t -> info (getId $ getCommandTokenOrThis t) 2230 "which is non-standard. Use builtin 'command -v' instead." + \t -> info (getId $ getCommandTokenOrThis t) 2230 "'which' is non-standard. Use builtin 'command -v' instead." prop_checkSudoRedirect1 = verify checkSudoRedirect "sudo echo 3 > /proc/file" prop_checkSudoRedirect2 = verify checkSudoRedirect "sudo cmd < input" From 5669eb22037980c5a6b74b0d420cb452990bcf88 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 11 Apr 2021 15:52:13 -0700 Subject: [PATCH 134/459] Make x-comparison warning default --- CHANGELOG.md | 2 +- src/ShellCheck/Analytics.hs | 10 ++-------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9eec957..f6d5a82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ - SC2264: Warn about wrapper functions that blatantly recurse - SC2265/SC2266: Warn when using & or | with test statements - SC2267: Warn when using xargs -i instead of -I -- Optional avoid-x-comparisons: Style warning SC2268 for `[ x$var = xval ]` +- SC2268: Warn about unnecessary x-comparisons like `[ x$var = xval ]` ### Fixed - SC1072/SC1073 now respond to disable annotations, though ignoring parse errors diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 9035e04..b53aee1 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -196,6 +196,7 @@ nodeChecks = [ ,checkAssignToSelf ,checkEqualsInCommand ,checkSecondArgIsComparison + ,checkComparisonWithLeadingX ] optionalChecks = map fst optionalTreeChecks @@ -243,13 +244,6 @@ optionalTreeChecks = [ cdPositive = "echo $VAR", cdNegative = "VAR=hello; echo $VAR" }, checkUnassignedReferences' True) - - ,(newCheckDescription { - cdName = "avoid-x-comparisons", - cdDescription = "Warn about 'x'-prefix in comparisons", - cdPositive = "[ \"x$var\" = xval ]", - cdNegative = "[ \"$var\" = val ]" - }, nodeChecksToTreeCheck [checkComparisonWithLeadingX]) ] optionalCheckMap :: Map.Map String (Parameters -> Token -> [TokenComment]) @@ -4006,7 +4000,7 @@ checkComparisonWithLeadingX params t = check lhs rhs _ -> return () where - msg = "Avoid outdated x-prefix in comparisons as it no longer serves a purpose." + msg = "Avoid x-prefix in comparisons as it no longer serves a purpose." check lhs rhs = sequence_ $ do l <- fixLeadingX lhs r <- fixLeadingX rhs From cff3e22911f25283ccef0a23bdfdfaafe3ad7c40 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 19 Apr 2021 14:31:13 -0700 Subject: [PATCH 135/459] Stable version v0.7.2 This release is dedicated to ethanol, for keeping COVID-19 off both our hands and our minds. --- CHANGELOG.md | 4 ++-- ShellCheck.cabal | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f6d5a82..4ff81cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## Git +## v0.7.2 - 2021-04-19 ### Added - `disable` directives can now be a range, e.g. `disable=SC3000-SC4000` - SC1143: Warn about line continuations in comments @@ -22,7 +22,7 @@ - POSIX/dash unsupported feature warnings now have individual SC3xxx codes - SC1090: A leading `$x/` or `$(x)/` is now treated as `./` when locating files - SC2154: Variables appearing in -z/-n tests are no longer considered unassigned -- SC2270-SC2285: Improved warnings about misused =, e.g. `${var}=42` +- SC2270-SC2285: Improved warnings about misused `=`, e.g. `${var}=42` ## v0.7.1 - 2020-04-04 diff --git a/ShellCheck.cabal b/ShellCheck.cabal index 2254c02..b474f47 100644 --- a/ShellCheck.cabal +++ b/ShellCheck.cabal @@ -1,5 +1,5 @@ Name: ShellCheck -Version: 0.7.1 +Version: 0.7.2 Synopsis: Shell script analysis tool License: GPL-3 License-file: LICENSE From aaa355472033b56ce56c10002ffd711feb5184ee Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 19 Apr 2021 16:40:25 -0700 Subject: [PATCH 136/459] Post-release CHANGELOG update --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ff81cf..af4b5a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +## Git +### Added + +### Fixed + +### Changed + + ## v0.7.2 - 2021-04-19 ### Added - `disable` directives can now be a range, e.g. `disable=SC3000-SC4000` From 2f26600653a26e890c850b64f8755bb5f5f302b7 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 19 Apr 2021 17:19:21 -0700 Subject: [PATCH 137/459] Update Cabal version for Hackage --- ShellCheck.cabal | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ShellCheck.cabal b/ShellCheck.cabal index b474f47..766b1d6 100644 --- a/ShellCheck.cabal +++ b/ShellCheck.cabal @@ -8,7 +8,7 @@ Author: Vidar Holen Maintainer: vidar@vidarholen.net Homepage: https://www.shellcheck.net/ Build-Type: Simple -Cabal-Version: >= 1.8 +Cabal-Version: >= 1.10 Bug-reports: https://github.com/koalaman/shellcheck/issues Description: The goals of ShellCheck are: @@ -83,6 +83,7 @@ library ShellCheck.Regex other-modules: Paths_ShellCheck + default-language: Haskell98 executable shellcheck if impl(ghc < 8.0) @@ -103,6 +104,7 @@ executable shellcheck QuickCheck >= 2.7.4, regex-tdfa, ShellCheck + default-language: Haskell98 main-is: shellcheck.hs test-suite test-shellcheck @@ -122,5 +124,6 @@ test-suite test-shellcheck QuickCheck >= 2.7.4, regex-tdfa, ShellCheck + default-language: Haskell98 main-is: test/shellcheck.hs From d47f3ff9866fde463cf318314387776608b799da Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 19 Apr 2021 22:46:35 -0700 Subject: [PATCH 138/459] Add wait between GitHub and Docker to allow replication --- .github/workflows/build.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 707b9a2..0acece2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -113,6 +113,10 @@ jobs: export TAGS="$(cat source/tags)" ./.github_deploy + - name: Waiting for GitHub to replicate uploaded releases + run: | + sleep 300 + - name: Upload to Docker Hub env: DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} From 9e60b3ea841bcaf48780bfcfc2e44aa6563a62de Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 22 Apr 2021 22:17:51 -0700 Subject: [PATCH 139/459] Fix haddock failures (fixes #2216) --- src/ShellCheck/Analytics.hs | 12 ++++++------ test/buildtest | 2 ++ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index b53aee1..b943cbc 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -4155,11 +4155,11 @@ checkEqualsInCommand params originalToken = _ | "===" `isPrefixOf` s -> borderMsg (getId originalToken) _ -> prefixMsg (getId cmd) - -- $var==42 + -- '$var==42' _ | "==" `isInfixOf` s -> badComparisonMsg (getId cmd) - -- ${foo[x]}=42 and $foo=42 + -- '${foo[x]}=42' and '$foo=42' [T_DollarBraced id braced l] | "=" `isPrefixOf` s -> do let variableStr = concat $ oversimplify l let variableReference = getBracedReference variableStr @@ -4172,22 +4172,22 @@ checkEqualsInCommand params originalToken = && "]" `isSuffixOf` variableModifier case () of - -- $foo=bar should already have caused a parse-time SC1066 + -- '$foo=bar' should already have caused a parse-time SC1066 -- _ | not braced && isPlain -> -- return () _ | variableStr == "" -> -- Don't try to fix ${}=foo genericMsg (getId cmd) - -- $#=42 or ${#var}=42 + -- '$#=42' or '${#var}=42' _ | "#" `isPrefixOf` variableStr -> genericMsg (getId cmd) - -- ${0}=42 + -- '${0}=42' _ | variableStr == "0" -> assign0Msg id $ fixWith [replaceToken id params "BASH_ARGV0"] - -- $2=2 + -- '$2=2' _ | isPositional -> positionalMsg id diff --git a/test/buildtest b/test/buildtest index 68bd048..1d194fc 100755 --- a/test/buildtest +++ b/test/buildtest @@ -29,6 +29,8 @@ cabal build || die "build failed" cabal test || die "test failed" +cabal haddock || + die "haddock failed" sc="$(find . -name shellcheck -type f -perm -111)" [ -x "$sc" ] || die "Can't find executable" From fe25a2b00e79a9be4371496541b5b15b9aa554bc Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 24 Apr 2021 17:08:10 -0700 Subject: [PATCH 140/459] Treat ${arr[*]} like $* for SC2048 --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 11 +++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index af4b5a0..e0295e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Fixed ### Changed +- SC2048: Warning about $\* now also applies to ${array[\*]} ## v0.7.2 - 2021-04-19 diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index b943cbc..97c0b27 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -820,10 +820,17 @@ checkShorthandIf _ _ = return () prop_checkDollarStar = verify checkDollarStar "for f in $*; do ..; done" prop_checkDollarStar2 = verifyNot checkDollarStar "a=$*" prop_checkDollarStar3 = verifyNot checkDollarStar "[[ $* = 'a b' ]]" +prop_checkDollarStar4 = verify checkDollarStar "for f in ${var[*]}; do ..; done" +prop_checkDollarStar5 = verify checkDollarStar "ls ${*//foo/bar}" +prop_checkDollarStar6 = verify checkDollarStar "ls ${var[*]%%.*}" checkDollarStar p t@(T_NormalWord _ [T_DollarBraced id _ l]) - | concat (oversimplify l) == "*" && - not (isStrictlyQuoteFree (parentMap p) t) = + | not (isStrictlyQuoteFree (parentMap p) t) = do + let str = concat (oversimplify l) + when ("*" `isPrefixOf` str) $ warn id 2048 "Use \"$@\" (with quotes) to prevent whitespace problems." + when ("[*]" `isPrefixOf` (getBracedModifier str)) $ + warn id 2048 "Use \"${array[@]}\" (with quotes) to prevent whitespace problems." + checkDollarStar _ _ = return () From 331e89be990547b6e21ad1b6e56065bcda1ba053 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 26 Apr 2021 10:33:36 -0700 Subject: [PATCH 141/459] Fix bad warning for ${#arr[*]}. Fixes #2218. --- src/ShellCheck/Analytics.hs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 97c0b27..e61c30a 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -823,12 +823,16 @@ prop_checkDollarStar3 = verifyNot checkDollarStar "[[ $* = 'a b' ]]" prop_checkDollarStar4 = verify checkDollarStar "for f in ${var[*]}; do ..; done" prop_checkDollarStar5 = verify checkDollarStar "ls ${*//foo/bar}" prop_checkDollarStar6 = verify checkDollarStar "ls ${var[*]%%.*}" +prop_checkDollarStar7 = verify checkDollarStar "ls ${*}" +prop_checkDollarStar8 = verifyNot checkDollarStar "ls ${#*}" +prop_checkDollarStar9 = verify checkDollarStar "ls ${arr[*]}" +prop_checkDollarStar10 = verifyNot checkDollarStar "ls ${#arr[*]}" checkDollarStar p t@(T_NormalWord _ [T_DollarBraced id _ l]) | not (isStrictlyQuoteFree (parentMap p) t) = do let str = concat (oversimplify l) when ("*" `isPrefixOf` str) $ warn id 2048 "Use \"$@\" (with quotes) to prevent whitespace problems." - when ("[*]" `isPrefixOf` (getBracedModifier str)) $ + when ("[*]" `isPrefixOf` (getBracedModifier str) && isVariableChar (headOrDefault '!' str)) $ warn id 2048 "Use \"${array[@]}\" (with quotes) to prevent whitespace problems." checkDollarStar _ _ = return () From ab369a35c91c61c6b915226e779f8611a5626f31 Mon Sep 17 00:00:00 2001 From: Jens Petersen Date: Tue, 18 May 2021 11:06:12 +0800 Subject: [PATCH 142/459] move readme to extra-doc-files and add changelog to releases --- ShellCheck.cabal | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ShellCheck.cabal b/ShellCheck.cabal index 766b1d6..9433f55 100644 --- a/ShellCheck.cabal +++ b/ShellCheck.cabal @@ -8,7 +8,7 @@ Author: Vidar Holen Maintainer: vidar@vidarholen.net Homepage: https://www.shellcheck.net/ Build-Type: Simple -Cabal-Version: >= 1.10 +Cabal-Version: 1.18 Bug-reports: https://github.com/koalaman/shellcheck/issues Description: The goals of ShellCheck are: @@ -22,9 +22,11 @@ Description: * To point out subtle caveats, corner cases and pitfalls, that may cause an advanced user's otherwise working script to fail under future circumstances. +Extra-Doc-Files: + README.md + CHANGELOG.md Extra-Source-Files: -- documentation - README.md shellcheck.1.md -- A script to build the man page using pandoc manpage @@ -126,4 +128,3 @@ test-suite test-shellcheck ShellCheck default-language: Haskell98 main-is: test/shellcheck.hs - From b61a7658d69c7f11be55f26f5d25186d4ac89cb6 Mon Sep 17 00:00:00 2001 From: Rebecca Cran Date: Mon, 24 May 2021 13:33:50 -0600 Subject: [PATCH 143/459] Fix typo in SC2006 message: "backticked" vs "backticks" --- src/ShellCheck/Analytics.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index e61c30a..da5f1f0 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1595,7 +1595,7 @@ prop_checkBackticks2 = verifyNot checkBackticks "echo $(foo)" prop_checkBackticks3 = verifyNot checkBackticks "echo `#inlined comment` foo" checkBackticks params (T_Backticked id list) | not (null list) = addComment $ - makeCommentWithFix StyleC id 2006 "Use $(...) notation instead of legacy backticked `...`." + makeCommentWithFix StyleC id 2006 "Use $(...) notation instead of legacy backticks `...`." (fixWith [replaceStart id params 1 "$(", replaceEnd id params 1 ")"]) checkBackticks _ _ = return () From 51009603037014d0536407f24e316acd7aa58580 Mon Sep 17 00:00:00 2001 From: Kamil Cukrowski Date: Wed, 26 May 2021 10:58:38 +0200 Subject: [PATCH 144/459] Add a comma to function characters Bash has very relaxed function name rules and a comma is also a valid character. This commit silences SC1036 check when a function name has a comma in its name. --- src/ShellCheck/Parser.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 3c16d5d..bf4fce4 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -66,7 +66,7 @@ doubleQuote = char '"' variableStart = upper <|> lower <|> oneOf "_" variableChars = upper <|> lower <|> digit <|> oneOf "_" -- Chars to allow in function names -functionChars = variableChars <|> oneOf ":+?-./^@" +functionChars = variableChars <|> oneOf ":+?-./^@," -- Chars to allow in functions using the 'function' keyword extendedFunctionChars = functionChars <|> oneOf "[]*=!" specialVariable = oneOf (concat specialVariables) From 163b2f12e2076a643e744774c8408370655bfc09 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 5 Jun 2021 18:16:22 -0700 Subject: [PATCH 145/459] Sanity check command names (fixes #2227) --- CHANGELOG.md | 2 ++ src/ShellCheck/Analytics.hs | 38 +++++++++++++++++++++++++++++++++++ src/ShellCheck/AnalyzerLib.hs | 4 ++++ 3 files changed, 44 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e0295e3..af762b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ ## Git ### Added +- SC2286-SC2288: Warn when command name ends in a symbol like `/.)'"` +- SC2289: Warn when command name contains tabs or linefeeds ### Fixed diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index e61c30a..7c44986 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -197,6 +197,7 @@ nodeChecks = [ ,checkEqualsInCommand ,checkSecondArgIsComparison ,checkComparisonWithLeadingX + ,checkCommandWithTrailingSymbol ] optionalChecks = map fst optionalTreeChecks @@ -4265,5 +4266,42 @@ checkSecondArgIsComparison _ t = T_NormalWord _ (x:_) -> getId x _ -> getId t + +prop_checkCommandWithTrailingSymbol1 = verify checkCommandWithTrailingSymbol "/" +prop_checkCommandWithTrailingSymbol2 = verify checkCommandWithTrailingSymbol "/foo/ bar/baz" +prop_checkCommandWithTrailingSymbol3 = verify checkCommandWithTrailingSymbol "/" +prop_checkCommandWithTrailingSymbol4 = verifyNot checkCommandWithTrailingSymbol "/*" +prop_checkCommandWithTrailingSymbol5 = verifyNot checkCommandWithTrailingSymbol "$foo/$bar" +prop_checkCommandWithTrailingSymbol6 = verify checkCommandWithTrailingSymbol "foo, bar" +prop_checkCommandWithTrailingSymbol7 = verifyNot checkCommandWithTrailingSymbol ". foo.sh" +prop_checkCommandWithTrailingSymbol8 = verifyNot checkCommandWithTrailingSymbol ": foo" +prop_checkCommandWithTrailingSymbol9 = verifyNot checkCommandWithTrailingSymbol "/usr/bin/python[23] file.py" + +checkCommandWithTrailingSymbol _ t = + case t of + T_SimpleCommand _ _ (cmd:_) -> + let str = fromJust $ getLiteralStringExt (\_ -> Just "x") cmd + last = lastOrDefault 'x' str + in + case str of + "." -> return () -- The . command + ":" -> return () -- The : command + " " -> return () -- Probably caught by SC1101 + "//" -> return () -- Probably caught by SC1127 + "" -> err (getId cmd) 2286 "This empty string is interpreted as a command name. Double check syntax (or use 'true' as a no-op)." + _ | last == '/' -> err (getId cmd) 2287 "This is interpreted as a command name ending with '/'. Double check syntax." + _ | last `elem` "\\.,([{<>}])#\"\'% " -> warn (getId cmd) 2288 ("This is interpreted as a command name ending with " ++ (format last) ++ ". Double check syntax.") + _ | '\t' `elem` str -> err (getId cmd) 2289 "This is interpreted as a command name containing a tab. Double check syntax." + _ | '\n' `elem` str -> err (getId cmd) 2289 "This is interpreted as a command name containing a linefeed. Double check syntax." + _ -> return () + _ -> return () + where + format x = + case x of + ' ' -> "space" + '\'' -> "apostrophe" + '\"' -> "doublequote" + x -> '\'' : x : "\'" + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 5b97e87..e4d76f8 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -877,6 +877,10 @@ getBracedModifier s = headOrDefault "" $ do headOrDefault _ (a:_) = a headOrDefault def _ = def +-- Get the last element or a default. Like `last` but safe. +lastOrDefault def [] = def +lastOrDefault _ list = last list + --- Get element n of a list, or Nothing. Like `!!` but safe. (!!!) list i = case drop i list of From 99f6554c9bd97d4815848b3aeba027ce196cbd41 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 18 Jul 2021 16:59:45 -0700 Subject: [PATCH 146/459] SC2181: Add '!' in suggestion as appropriate (fixes #2189) --- src/ShellCheck/Analytics.hs | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 3ae4d6a..88947a6 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -3118,24 +3118,29 @@ prop_checkReturnAgainstZero8 = verify checkReturnAgainstZero "(( $? ))" prop_checkReturnAgainstZero9 = verify checkReturnAgainstZero "(( ! $? ))" checkReturnAgainstZero _ token = case token of - TC_Binary id _ _ lhs rhs -> check lhs rhs - TA_Binary id _ lhs rhs -> check lhs rhs - TA_Unary id _ exp - | isExitCode exp -> message (getId exp) + TC_Binary id _ op lhs rhs -> check op lhs rhs + TA_Binary id op lhs rhs -> check op lhs rhs + TA_Unary id op exp + | isExitCode exp -> message (checksSuccessLhs op) (getId exp) TA_Sequence _ [exp] - | isExitCode exp -> message (getId exp) + | isExitCode exp -> message False (getId exp) _ -> return () where - check lhs rhs = + -- Is "$? op 0" trying to check if the command succeeded? + checksSuccessLhs op = not $ op `elem` ["-gt", "-ne", "!=", "!"] + -- Is "0 op $?" trying to check if the command succeeded? + checksSuccessRhs op = op `notElem` ["-ne", "!="] + check op lhs rhs = if isZero rhs && isExitCode lhs - then message (getId lhs) - else when (isZero lhs && isExitCode rhs) $ message (getId rhs) + then message (checksSuccessLhs op) (getId lhs) + else when (isZero lhs && isExitCode rhs) $ message (checksSuccessRhs op) (getId rhs) isZero t = getLiteralString t == Just "0" isExitCode t = case getWordParts t of [T_DollarBraced _ _ l] -> concat (oversimplify l) == "?" _ -> False - message id = style id 2181 "Check exit code directly with e.g. 'if mycmd;', not indirectly with $?." + message forSuccess id = style id 2181 $ + "Check exit code directly with e.g. 'if " ++ (if forSuccess then "" else "! ") ++ "mycmd;', not indirectly with $?." prop_checkRedirectedNowhere1 = verify checkRedirectedNowhere "> file" prop_checkRedirectedNowhere2 = verify checkRedirectedNowhere "> file | grep foo" From 9b077e28cbcf531f11176d4ac53688fc2ceb389b Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 21 Jul 2021 16:44:21 -0700 Subject: [PATCH 147/459] Add :/. to chars recognized for \alias suppression (fixes #2287) --- src/ShellCheck/Parser.hs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index bf4fce4..29dc4b0 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -2061,6 +2061,7 @@ prop_readSimpleCommand4 = isOk readSimpleCommand "typeset -a foo=(lol)" prop_readSimpleCommand5 = isOk readSimpleCommand "time if true; then echo foo; fi" prop_readSimpleCommand6 = isOk readSimpleCommand "time -p ( ls -l; )" prop_readSimpleCommand7 = isOk readSimpleCommand "\\ls" +prop_readSimpleCommand7b = isOk readSimpleCommand "\\:" prop_readSimpleCommand8 = isWarning readSimpleCommand "// Lol" prop_readSimpleCommand9 = isWarning readSimpleCommand "/* Lolbert */" prop_readSimpleCommand10 = isWarning readSimpleCommand "/**** Lolbert */" @@ -2321,7 +2322,7 @@ readCmdName = do -- Ignore alias suppression optional . try $ do char '\\' - lookAhead $ variableChars + lookAhead $ variableChars <|> oneOf ":." readCmdWord readCmdWord = do From 8be60028ef9ef52db907edb38f022802debe5b0b Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 22 Jul 2021 19:25:48 -0700 Subject: [PATCH 148/459] Don't warn when line starts with &> (fixes #2281) --- src/ShellCheck/Parser.hs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 29dc4b0..4a01775 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -1996,12 +1996,14 @@ readHereString = called "here string" $ do word <- readNormalWord return $ T_HereString id word +prop_readNewlineList1 = isOk readScript "&> /dev/null echo foo" readNewlineList = many1 ((linefeed <|> carriageReturn) `thenSkip` spacing) <* checkBadBreak where checkBadBreak = optional $ do pos <- getPosition try $ lookAhead (oneOf "|&") -- See if the next thing could be |, || or && + notFollowedBy2 (string "&>") -- Except &> or &>> which is valid parseProblemAt pos ErrorC 1133 "Unexpected start of line. If breaking lines, |/||/&& should be at the end of the previous one." readLineBreak = optional readNewlineList From 9eb63c97e6a5764b2fe0cb3e3e36cea5ade72a13 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 24 Jul 2021 13:07:05 -0700 Subject: [PATCH 149/459] Re-add warnings about 'declare var = value' (fixes #2279) --- CHANGELOG.md | 1 + src/ShellCheck/Checks/Commands.hs | 31 +++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index af762b6..92b65d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - SC2289: Warn when command name contains tabs or linefeeds ### Fixed +- SC2290: Warn about misused = in declare & co, which were not caught by SC2270+ ### Changed - SC2048: Warning about $\* now also applies to ${array[\*]} diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index d7452e2..0d8deae 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -95,6 +95,11 @@ commandChecks = [ ,checkSourceArgs ,checkChmodDashr ,checkXargsDashi + ,checkArgComparison "local" + ,checkArgComparison "declare" + ,checkArgComparison "export" + ,checkArgComparison "readonly" + ,checkArgComparison "typeset" ] optionalChecks = map fst optionalCommandChecks @@ -1143,5 +1148,31 @@ checkXargsDashi = CommandCheck (Basename "xargs") f return $ info (getId option) 2267 "GNU xargs -i is deprecated in favor of -I{}" parseOpts = getBsdOpts "0oprtxadR:S:J:L:l:n:P:s:e:E:i:I:" + +prop_checkArgComparison1 = verify (checkArgComparison "declare") "declare a = b" +prop_checkArgComparison2 = verify (checkArgComparison "declare") "declare a =b" +prop_checkArgComparison3 = verifyNot (checkArgComparison "declare") "declare a=b" +prop_checkArgComparison4 = verify (checkArgComparison "export") "export a +=b" +prop_checkArgComparison7 = verifyNot (checkArgComparison "declare") "declare -a +i foo" +-- This mirrors checkSecondArgIsComparison but for arguments to local/readonly/declare/export +checkArgComparison str = CommandCheck (Exactly str) wordsWithEqual + where + wordsWithEqual t = mapM_ check $ drop 1 $ arguments t + check arg = sequence_ $ do + str <- getLeadingUnquotedString arg + case str of + '=':_ -> + return $ err (headId arg) 2290 $ + "Remove spaces around = to assign." + '+':'=':_ -> + return $ err (headId arg) 2290 $ + "Remove spaces around += to append." + _ -> Nothing + + headId t = + case t of + T_NormalWord _ (x:_) -> getId x + _ -> getId t + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) From 0d58337cdd05d4d724ce1db4bcd6e1eb6c8721cf Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 25 Jul 2021 12:59:56 -0700 Subject: [PATCH 150/459] Don't warn about repeated range in [[ -v arr[xxx] ]] (fixes #2285) --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 13 ++++++++++++- src/ShellCheck/AnalyzerLib.hs | 6 +++--- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 92b65d7..62e516a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - SC2289: Warn when command name contains tabs or linefeeds ### Fixed +- SC2102 about repetitions in ranges no longer triggers on [[ -v arr[xx] ]] - SC2290: Warn about misused = in declare & co, which were not caught by SC2270+ ### Changed diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 88947a6..141529d 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2519,8 +2519,10 @@ prop_checkCharRangeGlob3 = verify checkCharRangeGlob "ls [10-15]" prop_checkCharRangeGlob4 = verifyNot checkCharRangeGlob "ls [a-zA-Z]" prop_checkCharRangeGlob5 = verifyNot checkCharRangeGlob "tr -d [a-zA-Z]" -- tr has 2060 prop_checkCharRangeGlob6 = verifyNot checkCharRangeGlob "[[ $x == [!!]* ]]" +prop_checkCharRangeGlob7 = verifyNot checkCharRangeGlob "[[ -v arr[keykey] ]]" +prop_checkCharRangeGlob8 = verifyNot checkCharRangeGlob "[[ arr[keykey] -gt 1 ]]" checkCharRangeGlob p t@(T_Glob id str) | - isCharClass str && not (isParamTo (parentMap p) "tr" t) = + isCharClass str && not (isParamTo (parentMap p) "tr" t) && not (isDereferenced t) = if ":" `isPrefixOf` contents && ":" `isSuffixOf` contents && contents /= ":" @@ -2537,6 +2539,15 @@ checkCharRangeGlob p t@(T_Glob id str) | '!':rest -> rest '^':rest -> rest x -> x + + -- Check if this is a dereferencing context like [[ -v array[operandhere] ]] + isDereferenced = fromMaybe False . msum . map isDereferencingOp . getPath (parentMap p) + isDereferencingOp t = + case t of + TC_Binary _ DoubleBracket str _ _ -> return $ isDereferencingBinaryOp str + TC_Unary _ _ str _ -> return $ str == "-v" + T_SimpleCommand {} -> return False + _ -> Nothing checkCharRangeGlob _ _ = return () diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index e4d76f8..dd6a604 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -737,7 +737,7 @@ getReferencedVariables parents t = TC_Unary id _ "-v" token -> getIfReference t token TC_Unary id _ "-R" token -> getIfReference t token TC_Binary id DoubleBracket op lhs rhs -> - if isDereferencing op + if isDereferencingBinaryOp op then concatMap (getIfReference t) [lhs, rhs] else [] @@ -771,12 +771,12 @@ getReferencedVariables parents t = when (isDigit h) $ fail "is a number" return (context, token, getBracedReference str) - isDereferencing = (`elem` ["-eq", "-ne", "-lt", "-le", "-gt", "-ge"]) - isArithmeticAssignment t = case getPath parents t of this: TA_Assignment _ "=" lhs _ :_ -> lhs == t _ -> False +isDereferencingBinaryOp = (`elem` ["-eq", "-ne", "-lt", "-le", "-gt", "-ge"]) + dataTypeFrom defaultType v = (case v of T_Array {} -> DataArray; _ -> defaultType) $ SourceFrom [v] From 364c33395e2f2d5500307f01989f70241c247d5a Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 25 Jul 2021 14:41:49 -0700 Subject: [PATCH 151/459] Don't print colors when $TERM is 'dumb' or unset (fixes #2260) --- CHANGELOG.md | 1 + src/ShellCheck/Formatter/Format.hs | 21 ++++++++++++--------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 62e516a..9e87af8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ ### Fixed - SC2102 about repetitions in ranges no longer triggers on [[ -v arr[xx] ]] - SC2290: Warn about misused = in declare & co, which were not caught by SC2270+ +- The flag --color=auto no longer outputs color when TERM is "dumb" or unset ### Changed - SC2048: Warning about $\* now also applies to ${array[\*]} diff --git a/src/ShellCheck/Formatter/Format.hs b/src/ShellCheck/Formatter/Format.hs index cb7dfe6..53b59a4 100644 --- a/src/ShellCheck/Formatter/Format.hs +++ b/src/ShellCheck/Formatter/Format.hs @@ -28,6 +28,7 @@ import Data.Array import Data.List import System.IO import System.Info +import System.Environment -- A formatter that carries along an arbitrary piece of data data Formatter = Formatter { @@ -68,12 +69,14 @@ makeNonVirtual comments contents = shouldOutputColor :: ColorOption -> IO Bool -shouldOutputColor colorOption = do - term <- hIsTerminalDevice stdout - let windows = "mingw" `isPrefixOf` os - let isUsableTty = term && not windows - let useColor = case colorOption of - ColorAlways -> True - ColorNever -> False - ColorAuto -> isUsableTty - return useColor +shouldOutputColor colorOption = + case colorOption of + ColorAlways -> return True + ColorNever -> return False + ColorAuto -> do + isTerminal <- hIsTerminalDevice stdout + term <- lookupEnv "TERM" + let windows = "mingw" `isPrefixOf` os + let dumbTerm = term `elem` [Just "dumb", Just "", Nothing] + let isUsableTty = isTerminal && not windows && not dumbTerm + return isUsableTty From 44471b73cc6090bbb944c1096b945d2da868d125 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 25 Jul 2021 17:31:13 -0700 Subject: [PATCH 152/459] Have SC2155 trigger on 'typeset' as well (fixes #2262) --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 26 ------------ src/ShellCheck/Checks/Commands.hs | 67 ++++++++++++++++++++++++++++--- 3 files changed, 63 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e87af8..b7a5fc4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Fixed - SC2102 about repetitions in ranges no longer triggers on [[ -v arr[xx] ]] +- SC2155 now recognizes `typeset` and local read-only `declare` statements - SC2290: Warn about misused = in declare & co, which were not caught by SC2270+ - The flag --color=auto no longer outputs color when TERM is "dumb" or unset diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 141529d..e5fd4cc 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -169,7 +169,6 @@ nodeChecks = [ ,checkTestArgumentSplitting ,checkConcatenatedDollarAt ,checkTildeInPath - ,checkMaskedReturns ,checkReadWithoutR ,checkLoopVariableReassignment ,checkTrailingBracket @@ -2970,31 +2969,6 @@ checkTestArgumentSplitting params t = err (getId token) 2255 "[ ] does not apply arithmetic evaluation. Evaluate with $((..)) for numbers, or use string comparator for strings." -prop_checkMaskedReturns1 = verify checkMaskedReturns "f() { local a=$(false); }" -prop_checkMaskedReturns2 = verify checkMaskedReturns "declare a=$(false)" -prop_checkMaskedReturns3 = verify checkMaskedReturns "declare a=\"`false`\"" -prop_checkMaskedReturns4 = verify checkMaskedReturns "readonly a=$(false)" -prop_checkMaskedReturns5 = verify checkMaskedReturns "readonly a=\"`false`\"" -prop_checkMaskedReturns6 = verifyNot checkMaskedReturns "declare a; a=$(false)" -prop_checkMaskedReturns7 = verifyNot checkMaskedReturns "f() { local -r a=$(false); }" -prop_checkMaskedReturns8 = verifyNot checkMaskedReturns "a=$(false); readonly a" -checkMaskedReturns _ t@(T_SimpleCommand id _ (cmd:rest)) = sequence_ $ do - name <- getCommandName t - guard $ name `elem` ["declare", "export", "readonly"] - || name == "local" && "r" `notElem` map snd (getAllFlags t) - return $ mapM_ checkArgs rest - where - checkArgs (T_Assignment id _ _ _ word) | any hasReturn $ getWordParts word = - warn id 2155 "Declare and assign separately to avoid masking return values." - checkArgs _ = return () - - hasReturn t = case t of - T_Backticked {} -> True - T_DollarExpansion {} -> True - _ -> False -checkMaskedReturns _ _ = return () - - prop_checkReadWithoutR1 = verify checkReadWithoutR "read -a foo" prop_checkReadWithoutR2 = verifyNot checkReadWithoutR "read -ar foo" prop_checkReadWithoutR3 = verifyNot checkReadWithoutR "read -t 0" diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 0d8deae..ec012c2 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -95,12 +95,12 @@ commandChecks = [ ,checkSourceArgs ,checkChmodDashr ,checkXargsDashi - ,checkArgComparison "local" - ,checkArgComparison "declare" - ,checkArgComparison "export" - ,checkArgComparison "readonly" - ,checkArgComparison "typeset" ] + ++ map checkArgComparison declaringCommands + ++ map checkMaskedReturns declaringCommands + +declaringCommands = ["local", "declare", "export", "readonly", "typeset"] + optionalChecks = map fst optionalCommandChecks optionalCommandChecks :: [(CheckDescription, CommandCheck)] @@ -1174,5 +1174,62 @@ checkArgComparison str = CommandCheck (Exactly str) wordsWithEqual T_NormalWord _ (x:_) -> getId x _ -> getId t + +prop_checkMaskedReturns1 = verify (checkMaskedReturns "local") "f() { local a=$(false); }" +prop_checkMaskedReturns2 = verify (checkMaskedReturns "declare") "declare a=$(false)" +prop_checkMaskedReturns3 = verify (checkMaskedReturns "declare") "declare a=\"`false`\"" +prop_checkMaskedReturns4 = verify (checkMaskedReturns "readonly") "readonly a=$(false)" +prop_checkMaskedReturns5 = verify (checkMaskedReturns "readonly") "readonly a=\"`false`\"" +prop_checkMaskedReturns6 = verifyNot (checkMaskedReturns "declare") "declare a; a=$(false)" +prop_checkMaskedReturns7 = verifyNot (checkMaskedReturns "local") "f() { local -r a=$(false); }" +prop_checkMaskedReturns8 = verifyNot (checkMaskedReturns "readonly") "a=$(false); readonly a" +prop_checkMaskedReturns9 = verify (checkMaskedReturns "typeset") "#!/bin/ksh\n f() { typeset -r x=$(false); }" +prop_checkMaskedReturns10 = verifyNot (checkMaskedReturns "typeset") "#!/bin/ksh\n function f { typeset -r x=$(false); }" +prop_checkMaskedReturns11 = verifyNot (checkMaskedReturns "typeset") "#!/bin/bash\n f() { typeset -r x=$(false); }" +prop_checkMaskedReturns12 = verify (checkMaskedReturns "typeset") "typeset -r x=$(false);" +prop_checkMaskedReturns13 = verify (checkMaskedReturns "typeset") "f() { typeset -g x=$(false); }" +prop_checkMaskedReturns14 = verify (checkMaskedReturns "declare") "declare x=${ false; }" +prop_checkMaskedReturns15 = verify (checkMaskedReturns "declare") "f() { declare x=$(false); }" +checkMaskedReturns str = CommandCheck (Exactly str) checkCmd + where + checkCmd t = do + path <- getPathM t + shell <- asks shellType + sequence_ $ do + name <- getCommandName t + + let flags = map snd (getAllFlags t) + let hasDashR = "r" `elem` flags + let hasDashG = "g" `elem` flags + let isInScopedFunction = any (isScopedFunction shell) path + + let isLocal = not hasDashG && isLocalInFunction name && isInScopedFunction + let isReadOnly = name == "readonly" || hasDashR + + -- Don't warn about local variables that are declared readonly, + -- because the workaround `local x; x=$(false); local -r x;` is annoying + guard . not $ isLocal && isReadOnly + + return $ mapM_ checkArgs $ arguments t + + checkArgs (T_Assignment id _ _ _ word) | any hasReturn $ getWordParts word = + warn id 2155 "Declare and assign separately to avoid masking return values." + checkArgs _ = return () + + isLocalInFunction = (`elem` ["local", "declare", "typeset"]) + isScopedFunction shell t = + case t of + T_BatsTest {} -> True + -- In ksh, only functions declared with 'function' have their own scope + T_Function _ (FunctionKeyword hasFunction) _ _ _ -> shell /= Ksh || hasFunction + _ -> False + + hasReturn t = case t of + T_Backticked {} -> True + T_DollarExpansion {} -> True + T_DollarBraceCommandExpansion {} -> True + _ -> False + + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) From 02e07625d178c5cda3b7c1e1ab90f3e68ed24837 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 25 Jul 2021 19:27:35 -0700 Subject: [PATCH 153/459] Warn about quoting in assignments to sh declaration utilities (fixes #1556) --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 14 ++++++++------ src/ShellCheck/AnalyzerLib.hs | 20 ++++++++++++++++---- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b7a5fc4..b43675a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ ### Changed - SC2048: Warning about $\* now also applies to ${array[\*]} +- Quote warnings are now emitted for declaration utilities in sh ## v0.7.2 - 2021-04-19 diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index e5fd4cc..6c4f04e 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -728,6 +728,7 @@ prop_checkUnquotedExpansions6 = verifyNot checkUnquotedExpansions "$(cmd)" prop_checkUnquotedExpansions7 = verifyNot checkUnquotedExpansions "cat << foo\n$(ls)\nfoo" prop_checkUnquotedExpansions8 = verifyNot checkUnquotedExpansions "set -- $(seq 1 4)" prop_checkUnquotedExpansions9 = verifyNot checkUnquotedExpansions "echo foo `# inline comment`" +prop_checkUnquotedExpansions10 = verify checkUnquotedExpansions "#!/bin/sh\nexport var=$(val)" checkUnquotedExpansions params = check where @@ -737,7 +738,7 @@ checkUnquotedExpansions params = check _ = return () tree = parentMap params examine t contents = - unless (null contents || shouldBeSplit t || isQuoteFree tree t || usedAsCommandName tree t) $ + unless (null contents || shouldBeSplit t || isQuoteFree (shellType params) tree t || usedAsCommandName tree t) $ warn (getId t) 2046 "Quote this to prevent word splitting." shouldBeSplit t = @@ -828,7 +829,7 @@ prop_checkDollarStar8 = verifyNot checkDollarStar "ls ${#*}" prop_checkDollarStar9 = verify checkDollarStar "ls ${arr[*]}" prop_checkDollarStar10 = verifyNot checkDollarStar "ls ${#arr[*]}" checkDollarStar p t@(T_NormalWord _ [T_DollarBraced id _ l]) - | not (isStrictlyQuoteFree (parentMap p) t) = do + | not (isStrictlyQuoteFree (shellType p) (parentMap p) t) = do let str = concat (oversimplify l) when ("*" `isPrefixOf` str) $ warn id 2048 "Use \"$@\" (with quotes) to prevent whitespace problems." @@ -849,7 +850,7 @@ prop_checkUnquotedDollarAt7 = verify checkUnquotedDollarAt "for f in ${var[@]}; prop_checkUnquotedDollarAt8 = verifyNot checkUnquotedDollarAt "echo \"${args[@]:+${args[@]}}\"" prop_checkUnquotedDollarAt9 = verifyNot checkUnquotedDollarAt "echo ${args[@]:+\"${args[@]}\"}" prop_checkUnquotedDollarAt10 = verifyNot checkUnquotedDollarAt "echo ${@+\"$@\"}" -checkUnquotedDollarAt p word@(T_NormalWord _ parts) | not $ isStrictlyQuoteFree (parentMap p) word = +checkUnquotedDollarAt p word@(T_NormalWord _ parts) | not $ isStrictlyQuoteFree (shellType p) (parentMap p) word = forM_ (find isArrayExpansion parts) $ \x -> unless (isQuotedAlternativeReference x) $ err (getId x) 2068 @@ -862,7 +863,7 @@ prop_checkConcatenatedDollarAt3 = verify checkConcatenatedDollarAt "echo $a$@" prop_checkConcatenatedDollarAt4 = verifyNot checkConcatenatedDollarAt "echo $@" prop_checkConcatenatedDollarAt5 = verifyNot checkConcatenatedDollarAt "echo \"${arr[@]}\"" checkConcatenatedDollarAt p word@T_NormalWord {} - | not $ isQuoteFree (parentMap p) word + | not $ isQuoteFree (shellType p) (parentMap p) word || null (drop 1 parts) = mapM_ for array where @@ -1891,6 +1892,7 @@ prop_checkSpacefulness40= verifyNotTree checkSpacefulness "a=$((x+1)); echo $a" prop_checkSpacefulness41= verifyNotTree checkSpacefulness "exec $1 --flags" prop_checkSpacefulness42= verifyNotTree checkSpacefulness "run $1 --flags" prop_checkSpacefulness43= verifyNotTree checkSpacefulness "$foo=42" +prop_checkSpacefulness44= verifyTree checkSpacefulness "#!/bin/dash\nexport var=$value" data SpaceStatus = SpaceSome | SpaceNone | SpaceEmpty deriving (Eq) instance Semigroup SpaceStatus where @@ -1972,7 +1974,7 @@ checkSpacefulness' onFind params t = isExpansion token && not (isArrayExpansion token) -- There's another warning for this && not (isCountingReference token) - && not (isQuoteFree parents token) + && not (isQuoteFree (shellType params) parents token) && not (isQuotedAlternativeReference token) && not (usedAsCommandName parents token) @@ -2090,7 +2092,7 @@ checkQuotesInLiterals params t = return $ case assignment of Just j | not (isParamTo parents "eval" expr) - && not (isQuoteFree parents expr) + && not (isQuoteFree (shellType params) parents expr) && not (squashesQuotes expr) -> [ makeComment WarningC j 2089 $ diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index dd6a604..dd957cd 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -287,14 +287,14 @@ isStrictlyQuoteFree = isQuoteFreeNode True isQuoteFree = isQuoteFreeNode False -isQuoteFreeNode strict tree t = +isQuoteFreeNode strict shell tree t = isQuoteFreeElement t || - headOrDefault False (mapMaybe isQuoteFreeContext (drop 1 $ getPath tree t)) + (fromMaybe False $ msum $ map isQuoteFreeContext $ drop 1 $ getPath tree t) where -- Is this node self-quoting in itself? isQuoteFreeElement t = case t of - T_Assignment {} -> True + T_Assignment {} -> assignmentIsQuoting t T_FdRedirect {} -> True _ -> False @@ -306,7 +306,7 @@ isQuoteFreeNode strict tree t = TC_Binary _ DoubleBracket _ _ _ -> return True TA_Sequence {} -> return True T_Arithmetic {} -> return True - T_Assignment {} -> return True + T_Assignment {} -> return $ assignmentIsQuoting t T_Redirecting {} -> return False T_DoubleQuoted _ _ -> return True T_DollarDoubleQuoted _ _ -> return True @@ -318,6 +318,18 @@ isQuoteFreeNode strict tree t = T_SelectIn {} -> return (not strict) _ -> Nothing + -- Check whether this assigment is self-quoting due to being a recognized + -- assignment passed to a Declaration Utility. This will soon be required + -- by POSIX: https://austingroupbugs.net/view.php?id=351 + assignmentIsQuoting t = shellParsesParamsAsAssignments || not (isAssignmentParamToCommand t) + shellParsesParamsAsAssignments = shell /= Sh + + -- Is this assignment a parameter to a command like export/typeset/etc? + isAssignmentParamToCommand (T_Assignment id _ _ _ _) = + case Map.lookup id tree of + Just (T_SimpleCommand _ _ (_:args)) -> id `elem` (map getId args) + _ -> False + -- Check if a token is a parameter to a certain command by name: -- Example: isParamTo (parentMap params) "sed" t isParamTo :: Map.Map Id Token -> String -> Token -> Bool From 4956b006ac46c0f03d74adbf51dbe573f5f3a859 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 25 Jul 2021 19:56:51 -0700 Subject: [PATCH 154/459] Fix broken test from previous commit --- src/ShellCheck/Analytics.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 6c4f04e..b65291a 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1892,7 +1892,7 @@ prop_checkSpacefulness40= verifyNotTree checkSpacefulness "a=$((x+1)); echo $a" prop_checkSpacefulness41= verifyNotTree checkSpacefulness "exec $1 --flags" prop_checkSpacefulness42= verifyNotTree checkSpacefulness "run $1 --flags" prop_checkSpacefulness43= verifyNotTree checkSpacefulness "$foo=42" -prop_checkSpacefulness44= verifyTree checkSpacefulness "#!/bin/dash\nexport var=$value" +prop_checkSpacefulness44= verifyTree checkSpacefulness "#!/bin/sh\nexport var=$value" data SpaceStatus = SpaceSome | SpaceNone | SpaceEmpty deriving (Eq) instance Semigroup SpaceStatus where From 754ab22d9485ad8d58b6a4943074066743f970ab Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 26 Jul 2021 18:29:55 -0700 Subject: [PATCH 155/459] Warn about unquoted blanks in echo (fixes #377) --- CHANGELOG.md | 1 + src/ShellCheck/Checks/Commands.hs | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b43675a..2d4e318 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ### Added - SC2286-SC2288: Warn when command name ends in a symbol like `/.)'"` - SC2289: Warn when command name contains tabs or linefeeds +- SC2291: Warn about repeated unquoted spaces between words in echo ### Fixed - SC2102 about repetitions in ranges no longer triggers on [[ -v arr[xx] ]] diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index ec012c2..1a08a7f 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -95,6 +95,7 @@ commandChecks = [ ,checkSourceArgs ,checkChmodDashr ,checkXargsDashi + ,checkUnquotedEchoSpaces ] ++ map checkArgComparison declaringCommands ++ map checkMaskedReturns declaringCommands @@ -1231,5 +1232,31 @@ checkMaskedReturns str = CommandCheck (Exactly str) checkCmd _ -> False +prop_checkUnquotedEchoSpaces1 = verify checkUnquotedEchoSpaces "echo foo bar" +prop_checkUnquotedEchoSpaces2 = verifyNot checkUnquotedEchoSpaces "echo foo" +prop_checkUnquotedEchoSpaces3 = verifyNot checkUnquotedEchoSpaces "echo foo bar" +prop_checkUnquotedEchoSpaces4 = verifyNot checkUnquotedEchoSpaces "echo 'foo bar'" +prop_checkUnquotedEchoSpaces5 = verifyNot checkUnquotedEchoSpaces "echo a > myfile.txt b" +prop_checkUnquotedEchoSpaces6 = verifyNot checkUnquotedEchoSpaces " echo foo\\\n bar" +checkUnquotedEchoSpaces = CommandCheck (Basename "echo") check + where + check t = do + let args = arguments t + m <- asks tokenPositions + redir <- getClosestCommandM t + sequence_ $ do + let positions = mapMaybe (\c -> Map.lookup (getId c) m) args + let pairs = zip positions (drop 1 positions) + (T_Redirecting _ redirTokens _) <- redir + let redirPositions = mapMaybe (\c -> fst <$> Map.lookup (getId c) m) redirTokens + guard $ any (hasSpacesBetween redirPositions) pairs + return $ info (getId t) 2291 "Quote repeated spaces to avoid them collapsing into one." + + hasSpacesBetween redirs ((a,b), (c,d)) = + posLine a == posLine d + && ((posColumn c) - (posColumn b)) >= 4 + && not (any (\x -> b < x && x < c) redirs) + + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) From c471e458224b449f73e9d2e9ad7fac670c262a08 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 26 Jul 2021 19:32:33 -0700 Subject: [PATCH 156/459] Allow printf/return/assignments after exec (fixes #2249) --- src/ShellCheck/Analytics.hs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index b65291a..f907ce3 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1705,6 +1705,7 @@ prop_checkSpuriousExec6 = verify checkSpuriousExec "exec foo > file; cmd" prop_checkSpuriousExec7 = verifyNot checkSpuriousExec "exec file; echo failed; exit 3" prop_checkSpuriousExec8 = verifyNot checkSpuriousExec "exec {origout}>&1- >tmp.log 2>&1; bar" prop_checkSpuriousExec9 = verify checkSpuriousExec "for file in rc.d/*; do exec \"$file\"; done" +prop_checkSpuriousExec10 = verifyNot checkSpuriousExec "exec file; r=$?; printf >&2 'failed\n'; return $r" checkSpuriousExec _ = doLists where doLists (T_Script _ _ cmds) = doList cmds False @@ -1720,7 +1721,8 @@ checkSpuriousExec _ = doLists stripCleanup = reverse . dropWhile cleanup . reverse cleanup (T_Pipeline _ _ [cmd]) = - isCommandMatch cmd (`elem` ["echo", "exit"]) + isCommandMatch cmd (`elem` ["echo", "exit", "printf", "return"]) + || isAssignment cmd cleanup _ = False doList = doList' . stripCleanup From fbc8d2cb2f8070f820c9337851bb97478e40e710 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 26 Jul 2021 20:48:47 -0700 Subject: [PATCH 157/459] Don't consider [ -n/-z/-v $var ] assignments for subshell modification (fixes #2217) --- src/ShellCheck/Analytics.hs | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index f907ce3..256c5b8 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1800,6 +1800,9 @@ prop_subshellAssignmentCheck17 = verifyNotTree subshellAssignmentCheck "foo=${ { prop_subshellAssignmentCheck18 = verifyTree subshellAssignmentCheck "( exec {n}>&2; ); echo $n" prop_subshellAssignmentCheck19 = verifyNotTree subshellAssignmentCheck "#!/bin/bash\nshopt -s lastpipe; echo a | read -r b; echo \"$b\"" prop_subshellAssignmentCheck20 = verifyTree subshellAssignmentCheck "@test 'foo' { a=1; }\n@test 'bar' { echo $a; }\n" +prop_subshellAssignmentCheck21 = verifyNotTree subshellAssignmentCheck "test1() { echo foo | if [[ $var ]]; then echo $var; fi; }; test2() { echo $var; }" +prop_subshellAssignmentCheck22 = verifyNotTree subshellAssignmentCheck "( [[ -n $foo || -z $bar ]] ); echo $foo $bar" +prop_subshellAssignmentCheck23 = verifyNotTree subshellAssignmentCheck "( export foo ); echo $foo" subshellAssignmentCheck params t = let flow = variableFlow params check = findSubshelled flow [("oops",[])] Map.empty @@ -1807,8 +1810,19 @@ subshellAssignmentCheck params t = findSubshelled [] _ _ = return () -findSubshelled (Assignment x@(_, _, str, _):rest) ((reason,scope):lol) deadVars = - findSubshelled rest ((reason, x:scope):lol) $ Map.insert str Alive deadVars +findSubshelled (Assignment x@(_, _, str, data_):rest) scopes@((reason,scope):restscope) deadVars = + if isTrueAssignment data_ + then findSubshelled rest ((reason, x:scope):restscope) $ Map.insert str Alive deadVars + else findSubshelled rest scopes deadVars + where + isTrueAssignment c = + case c of + DataString SourceChecked -> False + DataString SourceDeclaration -> False + DataArray SourceChecked -> False + DataArray SourceDeclaration -> False + _ -> True + findSubshelled (Reference (_, readToken, str):rest) scopes deadVars = do unless (shouldIgnore str) $ case Map.findWithDefault Alive str deadVars of Alive -> return () From fe81dc1c275f052856721bcac50f7dcf2e997c86 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 27 Jul 2021 18:53:30 -0700 Subject: [PATCH 158/459] Optionally suggest [[ over [ in Bash scripts (-o require-double-brackets) (fixes #887) --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 41 +++++++++++++++++++++++++++++++++++ src/ShellCheck/AnalyzerLib.hs | 3 ++- 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d4e318..272c694 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ - SC2286-SC2288: Warn when command name ends in a symbol like `/.)'"` - SC2289: Warn when command name contains tabs or linefeeds - SC2291: Warn about repeated unquoted spaces between words in echo +- SC2292: Suggest [[ over [ in Bash/Ksh scripts (optional) ### Fixed - SC2102 about repetitions in ranges no longer triggers on [[ -v arr[xx] ]] diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 256c5b8..f1c603e 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -244,6 +244,13 @@ optionalTreeChecks = [ cdPositive = "echo $VAR", cdNegative = "VAR=hello; echo $VAR" }, checkUnassignedReferences' True) + + ,(newCheckDescription { + cdName = "require-double-brackets", + cdDescription = "Require [[ and warn about [ in Bash/Ksh", + cdPositive = "[ -e /etc/issue ]", + cdNegative = "[[ -e /etc/issue ]]" + }, checkRequireDoubleBracket) ] optionalCheckMap :: Map.Map String (Parameters -> Token -> [TokenComment]) @@ -4311,5 +4318,39 @@ checkCommandWithTrailingSymbol _ t = '\"' -> "doublequote" x -> '\'' : x : "\'" + +prop_checkRequireDoubleBracket1 = verifyTree checkRequireDoubleBracket "[ -x foo ]" +prop_checkRequireDoubleBracket2 = verifyTree checkRequireDoubleBracket "[ foo -o bar ]" +prop_checkRequireDoubleBracket3 = verifyNotTree checkRequireDoubleBracket "#!/bin/sh\n[ -x foo ]" +prop_checkRequireDoubleBracket4 = verifyNotTree checkRequireDoubleBracket "[[ -x foo ]]" +checkRequireDoubleBracket params = + if isBashLike params + then nodeChecksToTreeCheck [check] params + else const [] + where + check _ t = case t of + T_Condition id SingleBracket _ -> + styleWithFix id 2292 "Prefer [[ ]] over [ ] for tests in Bash/Ksh." (fixFor t) + _ -> return () + + fixFor t = fixWith $ + if isSimple t + then + [ + replaceStart (getId t) params 0 "[", + replaceEnd (getId t) params 0 "]" + ] + else [] + + -- We don't tag operators like < and -o well enough to replace them, + -- so just handle the simple cases. + isSimple t = case t of + T_Condition _ _ s -> isSimple s + TC_Binary _ _ op _ _ -> not $ any (\x -> x `elem` op) "<>" + TC_Unary {} -> True + TC_Nullary {} -> True + _ -> False + + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index dd957cd..2e19a6e 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -178,7 +178,8 @@ makeCommentWithFix :: Severity -> Id -> Code -> String -> Fix -> TokenComment makeCommentWithFix severity id code str fix = let comment = makeComment severity id code str withFix = comment { - tcFix = Just fix + -- If fix is empty, pretend it wasn't there. + tcFix = if null (fixReplacements fix) then Nothing else Just fix } in force withFix From e33146d530986ded1c82e5bf8ab7dd3a36924e8e Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 29 Jul 2021 20:51:19 -0700 Subject: [PATCH 159/459] Avoid trigger SC2181 on composite $? checks (fixes #1167) --- CHANGELOG.md | 2 ++ src/ShellCheck/Analytics.hs | 61 ++++++++++++++++++++++++++++++++++--- 2 files changed, 59 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 272c694..f21857b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,11 +8,13 @@ ### Fixed - SC2102 about repetitions in ranges no longer triggers on [[ -v arr[xx] ]] - SC2155 now recognizes `typeset` and local read-only `declare` statements +- SC2181 now tries to avoid triggering for error handling functions - SC2290: Warn about misused = in declare & co, which were not caught by SC2270+ - The flag --color=auto no longer outputs color when TERM is "dumb" or unset ### Changed - SC2048: Warning about $\* now also applies to ${array[\*]} +- SC2181 now only triggers on single condition tests like `[ $? = 0 ]`. - Quote warnings are now emitted for declaration utilities in sh diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index f1c603e..d036c40 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -3126,20 +3126,71 @@ prop_checkReturnAgainstZero6 = verifyNot checkReturnAgainstZero "[[ $R -eq 0 ]]" prop_checkReturnAgainstZero7 = verify checkReturnAgainstZero "(( $? == 0 ))" prop_checkReturnAgainstZero8 = verify checkReturnAgainstZero "(( $? ))" prop_checkReturnAgainstZero9 = verify checkReturnAgainstZero "(( ! $? ))" -checkReturnAgainstZero _ token = +prop_checkReturnAgainstZero10 = verifyNot checkReturnAgainstZero "x=$(( $? > 0 ))" +prop_checkReturnAgainstZero11 = verify checkReturnAgainstZero "(( ! ! ! $? ))" +prop_checkReturnAgainstZero12 = verify checkReturnAgainstZero "[ ! $? -eq 0 ]" +prop_checkReturnAgainstZero13 = verifyNot checkReturnAgainstZero "(( ! $? && $? > 42))" +prop_checkReturnAgainstZero14 = verifyNot checkReturnAgainstZero "[[ -e foo || $? -eq 0 ]]" +prop_checkReturnAgainstZero15 = verifyNot checkReturnAgainstZero "(( $?, n=1 ))" +prop_checkReturnAgainstZero16 = verifyNot checkReturnAgainstZero "(( $? || $? == 4 ))" +prop_checkReturnAgainstZero17 = verifyNot checkReturnAgainstZero "(( $? + 0 ))" +prop_checkReturnAgainstZero18 = verifyNot checkReturnAgainstZero "f() { if [ $? -eq 0 ]; then :; fi; }" +prop_checkReturnAgainstZero19 = verifyNot checkReturnAgainstZero "f() ( [ $? -eq 0 ] || exit 42; )" +prop_checkReturnAgainstZero20 = verify checkReturnAgainstZero "f() { if :; then x; [ $? -eq 0 ] && exit; fi; }" +prop_checkReturnAgainstZero21 = verify checkReturnAgainstZero "(( ( $? ) ))" +prop_checkReturnAgainstZero22 = verify checkReturnAgainstZero "[[ ( $? -eq 0 ) ]]" +checkReturnAgainstZero params token = case token of TC_Binary id _ op lhs rhs -> check op lhs rhs - TA_Binary id op lhs rhs -> check op lhs rhs - TA_Unary id op exp + TA_Binary id op lhs rhs + | op `elem` [">", "<", ">=", "<=", "==", "!="] -> check op lhs rhs + TA_Unary id op@"!" exp | isExitCode exp -> message (checksSuccessLhs op) (getId exp) TA_Sequence _ [exp] | isExitCode exp -> message False (getId exp) _ -> return () where + -- We don't want to warn about composite expressions like + -- [[ $? -eq 0 || $? -eq 4 ]] since these can be annoying to rewrite. + isOnlyTestInCommand t = + case getPath (parentMap params) t of + _:(T_Condition {}):_ -> True + _:(T_Arithmetic {}):_ -> True + _:(TA_Sequence _ [_]):(T_Arithmetic {}):_ -> True + + -- Some negations and groupings are also fine + _:next@(TC_Unary _ _ "!" _):_ -> isOnlyTestInCommand next + _:next@(TA_Unary _ "!" _):_ -> isOnlyTestInCommand next + _:next@(TC_Group {}):_ -> isOnlyTestInCommand next + _:next@(TA_Sequence _ [_]):_ -> isOnlyTestInCommand next + _ -> False + + -- TODO: Do better $? tracking and filter on whether + -- the target command is in the same function + getFirstCommandInFunction = f + where + f t = case t of + T_Function _ _ _ _ x -> f x + T_BraceGroup _ (x:_) -> f x + T_Subshell _ (x:_) -> f x + T_Annotation _ _ x -> f x + T_AndIf _ x _ -> f x + T_OrIf _ x _ -> f x + T_Pipeline _ _ (x:_) -> f x + T_Redirecting _ _ (T_IfExpression _ (((x:_),_):_) _) -> f x + x -> x + + isFirstCommandInFunction = fromMaybe False $ do + let path = getPath (parentMap params) token + func <- listToMaybe $ filter isFunction path + cmd <- getClosestCommand (parentMap params) token + return $ getId cmd == getId (getFirstCommandInFunction func) + -- Is "$? op 0" trying to check if the command succeeded? checksSuccessLhs op = not $ op `elem` ["-gt", "-ne", "!=", "!"] -- Is "0 op $?" trying to check if the command succeeded? checksSuccessRhs op = op `notElem` ["-ne", "!="] + check op lhs rhs = if isZero rhs && isExitCode lhs then message (checksSuccessLhs op) (getId lhs) @@ -3149,9 +3200,11 @@ checkReturnAgainstZero _ token = case getWordParts t of [T_DollarBraced _ _ l] -> concat (oversimplify l) == "?" _ -> False - message forSuccess id = style id 2181 $ + + message forSuccess id = when (isOnlyTestInCommand token && not isFirstCommandInFunction) $ style id 2181 $ "Check exit code directly with e.g. 'if " ++ (if forSuccess then "" else "! ") ++ "mycmd;', not indirectly with $?." + prop_checkRedirectedNowhere1 = verify checkRedirectedNowhere "> file" prop_checkRedirectedNowhere2 = verify checkRedirectedNowhere "> file | grep foo" prop_checkRedirectedNowhere3 = verify checkRedirectedNowhere "grep foo | > bar" From a44f3edb1424e47fa9be790dbd3e72e2ff2904bd Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Fri, 30 Jul 2021 18:46:19 -0700 Subject: [PATCH 160/459] Warn about eval'ing arrays --- CHANGELOG.md | 1 + src/ShellCheck/AnalyzerLib.hs | 2 ++ src/ShellCheck/Checks/Commands.hs | 21 +++++++++++++++++++++ 3 files changed, 24 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f21857b..fe8d321 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - SC2289: Warn when command name contains tabs or linefeeds - SC2291: Warn about repeated unquoted spaces between words in echo - SC2292: Suggest [[ over [ in Bash/Ksh scripts (optional) +- SC2293/SC2294: Warn when calling `eval` with arrays ### Fixed - SC2102 about repetitions in ranges no longer triggers on [[ -v arr[xx] ]] diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 2e19a6e..beb4e5c 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -872,6 +872,8 @@ getBracedReference s = fromMaybe s $ prop_getBracedModifier1 = getBracedModifier "foo:bar:baz" == ":bar:baz" prop_getBracedModifier2 = getBracedModifier "!var:-foo" == ":-foo" prop_getBracedModifier3 = getBracedModifier "foo[bar]" == "[bar]" +prop_getBracedModifier4 = getBracedModifier "foo[@]@Q" == "[@]@Q" +prop_getBracedModifier5 = getBracedModifier "@@Q" == "@Q" getBracedModifier s = headOrDefault "" $ do let var = getBracedReference s a <- dropModifier s diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 1a08a7f..366438d 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -96,6 +96,7 @@ commandChecks = [ ,checkChmodDashr ,checkXargsDashi ,checkUnquotedEchoSpaces + ,checkEvalArray ] ++ map checkArgComparison declaringCommands ++ map checkMaskedReturns declaringCommands @@ -1258,5 +1259,25 @@ checkUnquotedEchoSpaces = CommandCheck (Basename "echo") check && not (any (\x -> b < x && x < c) redirs) +prop_checkEvalArray1 = verify checkEvalArray "eval $@" +prop_checkEvalArray2 = verify checkEvalArray "eval \"${args[@]}\"" +prop_checkEvalArray3 = verify checkEvalArray "eval \"${args[@]@Q}\"" +prop_checkEvalArray4 = verifyNot checkEvalArray "eval \"${args[*]@Q}\"" +prop_checkEvalArray5 = verifyNot checkEvalArray "eval \"$*\"" +checkEvalArray = CommandCheck (Exactly "eval") (mapM_ check . concatMap getWordParts . arguments) + where + check t = + when (isArrayExpansion t) $ + if isEscaped t + then style (getId t) 2293 "When eval'ing @Q-quoted words, use * rather than @ as the index." + else warn (getId t) 2294 "eval negates the benefit of arrays. Drop eval to preserve whitespace/symbols (or eval as string)." + + isEscaped q = + case q of + -- Match ${arr[@]@Q} and ${@@Q} and such + T_DollarBraced _ _ l -> 'Q' `elem` getBracedModifier (concat $ oversimplify l) + _ -> False + + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) From b939f86331f6343e1fb7f2fd84a26f84f608fd1b Mon Sep 17 00:00:00 2001 From: Yancharuk Alexander Date: Sat, 31 Jul 2021 06:24:20 +0300 Subject: [PATCH 161/459] Minor changes in README It's a recommended practice to use apt instead apt-get: >apt is a second command-line based front end provided by APT which overcomes some design mistakes of apt-get. https://debian-handbook.info/browse/stable/sect.apt-get.html Also added sudo for commands needed root privileges. --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index fe16fb2..551f7a8 100644 --- a/README.md +++ b/README.md @@ -143,7 +143,7 @@ On systems with Stack (installs to `~/.local/bin`): On Debian based distros: - apt-get install shellcheck + sudo apt install shellcheck On Arch Linux based distros: @@ -157,8 +157,8 @@ On Gentoo based distros: On EPEL based distros: - yum -y install epel-release - yum install ShellCheck + sudo yum -y install epel-release + sudo yum install ShellCheck On Fedora based distros: @@ -166,7 +166,7 @@ On Fedora based distros: On FreeBSD: - pkg install hs-ShellCheck + sudo pkg install hs-ShellCheck On macOS (OS X) with Homebrew: From 2f61b17518f51971eaee9cdb79631625bc3e6cae Mon Sep 17 00:00:00 2001 From: Yancharuk Alexander Date: Mon, 2 Aug 2021 19:09:24 +0300 Subject: [PATCH 162/459] Review fixes in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 551f7a8..dcfde31 100644 --- a/README.md +++ b/README.md @@ -166,7 +166,7 @@ On Fedora based distros: On FreeBSD: - sudo pkg install hs-ShellCheck + pkg install hs-ShellCheck On macOS (OS X) with Homebrew: From cf8066c07c063aca986d441583a4ee291dab74ae Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 3 Aug 2021 12:54:03 -0700 Subject: [PATCH 163/459] SC2295 Warn about unquoted variables in PE patterns (fixes #2290) --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 33 ++++++++++++++++++++++++++++++--- src/ShellCheck/AnalyzerLib.hs | 2 ++ 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fe8d321..2c99322 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ - SC2291: Warn about repeated unquoted spaces between words in echo - SC2292: Suggest [[ over [ in Bash/Ksh scripts (optional) - SC2293/SC2294: Warn when calling `eval` with arrays +- SC2295: Warn about "${x#$y}" treating $y as a pattern when not quoted ### Fixed - SC2102 about repetitions in ranges no longer triggers on [[ -v arr[xx] ]] diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index d036c40..a415ca2 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -197,6 +197,7 @@ nodeChecks = [ ,checkSecondArgIsComparison ,checkComparisonWithLeadingX ,checkCommandWithTrailingSymbol + ,checkUnquotedParameterExpansionPattern ] optionalChecks = map fst optionalTreeChecks @@ -388,7 +389,7 @@ replaceToken id params r = repInsertionPoint = InsertBefore } -surroundWidth id params s = fixWith [replaceStart id params 0 s, replaceEnd id params 0 s] +surroundWith id params s = fixWith [replaceStart id params 0 s, replaceEnd id params 0 s] fixWith fixes = newFix { fixReplacements = fixes } prop_checkEchoWc3 = verify checkEchoWc "n=$(echo $foo | wc -c)" @@ -1977,7 +1978,7 @@ quotesMayConflictWithSC2281 params t = (getId t) == (getId me) && (parentId == getId cmd) _ -> False -addDoubleQuotesAround params token = (surroundWidth (getId token) params "\"") +addDoubleQuotesAround params token = (surroundWith (getId token) params "\"") checkSpacefulness' :: (SpaceStatus -> Token -> String -> Writer [TokenComment] ()) -> Parameters -> Token -> [TokenComment] @@ -3274,7 +3275,7 @@ checkArrayAssignmentIndices params root = T_Literal id str <- parts let (before, after) = break ('=' ==) str guard $ all isDigit before && not (null after) - return $ warnWithFix id 2191 "The = here is literal. To assign by index, use ( [index]=value ) with no spaces. To keep as literal, quote it." (surroundWidth id params "\"") + return $ warnWithFix id 2191 "The = here is literal. To assign by index, use ( [index]=value ) with no spaces. To keep as literal, quote it." (surroundWith id params "\"") in if null literalEquals && isAssociative then warn (getId t) 2190 "Elements in associative arrays need index, e.g. array=( [index]=value ) ." @@ -4405,5 +4406,31 @@ checkRequireDoubleBracket params = _ -> False +prop_checkUnquotedParameterExpansionPattern1 = verify checkUnquotedParameterExpansionPattern "echo \"${var#$x}\"" +prop_checkUnquotedParameterExpansionPattern2 = verify checkUnquotedParameterExpansionPattern "echo \"${var%%$(x)}\"" +prop_checkUnquotedParameterExpansionPattern3 = verifyNot checkUnquotedParameterExpansionPattern "echo \"${var[#$x]}\"" +prop_checkUnquotedParameterExpansionPattern4 = verifyNot checkUnquotedParameterExpansionPattern "echo \"${var%\"$x\"}\"" + +checkUnquotedParameterExpansionPattern params x = + case x of + T_DollarBraced _ True word@(T_NormalWord _ (T_Literal _ s : rest@(_:_))) -> do + let modifier = getBracedModifier $ concat $ oversimplify word + when ("%" `isPrefixOf` modifier || "#" `isPrefixOf` modifier) $ + mapM_ check rest + _ -> return () + where + check t = + case t of + T_DollarBraced {} -> inform t + T_DollarExpansion {} -> inform t + T_Backticked {} -> inform t + _ -> return () + + inform t = + infoWithFix (getId t) 2295 + "Expansions inside ${..} need to be quoted separately, otherwise they match as patterns." $ + surroundWith (getId t) params "\"" + + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index beb4e5c..439b48f 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -167,6 +167,8 @@ errWithFix :: MonadWriter [TokenComment] m => Id -> Code -> String -> Fix -> m ( errWithFix = addCommentWithFix ErrorC warnWithFix :: MonadWriter [TokenComment] m => Id -> Code -> String -> Fix -> m () warnWithFix = addCommentWithFix WarningC +infoWithFix :: MonadWriter [TokenComment] m => Id -> Code -> String -> Fix -> m () +infoWithFix = addCommentWithFix InfoC styleWithFix :: MonadWriter [TokenComment] m => Id -> Code -> String -> Fix -> m () styleWithFix = addCommentWithFix StyleC From 378c9a2f2c396ac1323ebfcc599642a7e46ac133 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 3 Aug 2021 13:45:09 -0700 Subject: [PATCH 164/459] Switch build status badge from TravisCI to GitHub --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index dcfde31..bc0402c 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ -[![Build Status](https://travis-ci.org/koalaman/shellcheck.svg?branch=master)](https://travis-ci.org/koalaman/shellcheck) +[![Build Status](https://github.com/koalaman/shellcheck/actions/workflows/build.yml/badge.svg)](https://github.com/koalaman/shellcheck/actions/workflows/build.yml) + # ShellCheck - A shell script static analysis tool From 4dd762253f3aba35fe7a5070ab9395f062078637 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 3 Aug 2021 13:52:06 -0700 Subject: [PATCH 165/459] Remove defunct SonarQube plugin link (fixes #2292) --- README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/README.md b/README.md index bc0402c..a155b83 100644 --- a/README.md +++ b/README.md @@ -113,10 +113,6 @@ Services and platforms that have ShellCheck pre-installed and ready to use: * [CircleCI](https://circleci.com) via the [ShellCheck Orb](https://circleci.com/orbs/registry/orb/circleci/shellcheck) * [Github](https://github.com/features/actions) (only Linux) -Services and platforms with third party plugins: - -* [SonarQube](https://www.sonarqube.org/) through [sonar-shellcheck-plugin](https://github.com/emerald-squad/sonar-shellcheck-plugin) - Most other services, including [GitLab](https://about.gitlab.com/), let you install ShellCheck yourself, either through the system's package manager (see [Installing](#installing)), or by downloading and unpacking a [binary release](#installing-a-pre-compiled-binary). From e5745568e8b5ac18c2aa35a24aa6562c94380c74 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 8 Aug 2021 15:48:50 -0700 Subject: [PATCH 166/459] Extend warnings about spaces around = to 'let' --- src/ShellCheck/Checks/Commands.hs | 19 +++++++++++++++---- src/ShellCheck/Parser.hs | 2 +- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 366438d..c0cd9a3 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -101,7 +101,7 @@ commandChecks = [ ++ map checkArgComparison declaringCommands ++ map checkMaskedReturns declaringCommands -declaringCommands = ["local", "declare", "export", "readonly", "typeset"] +declaringCommands = ["local", "declare", "export", "readonly", "typeset", "let"] optionalChecks = map fst optionalCommandChecks @@ -1156,11 +1156,13 @@ prop_checkArgComparison2 = verify (checkArgComparison "declare") "declare a =b" prop_checkArgComparison3 = verifyNot (checkArgComparison "declare") "declare a=b" prop_checkArgComparison4 = verify (checkArgComparison "export") "export a +=b" prop_checkArgComparison7 = verifyNot (checkArgComparison "declare") "declare -a +i foo" +prop_checkArgComparison8 = verify (checkArgComparison "let") "let x = 0" -- This mirrors checkSecondArgIsComparison but for arguments to local/readonly/declare/export -checkArgComparison str = CommandCheck (Exactly str) wordsWithEqual +checkArgComparison cmd = CommandCheck (Exactly cmd) wordsWithEqual where - wordsWithEqual t = mapM_ check $ drop 1 $ arguments t - check arg = sequence_ $ do + wordsWithEqual t = mapM_ check $ arguments t + check arg = do + sequence_ $ do str <- getLeadingUnquotedString arg case str of '=':_ -> @@ -1171,6 +1173,15 @@ checkArgComparison str = CommandCheck (Exactly str) wordsWithEqual "Remove spaces around += to append." _ -> Nothing + -- 'let' is parsed as a sequence of arithmetic expansions, + -- so we want the additional warning for "x=" + when (cmd == "let") $ sequence_ $ do + token <- getTrailingUnquotedLiteral arg + str <- getLiteralString token + guard $ "=" `isSuffixOf` str + return $ err (getId token) 2290 $ + "Remove spaces around = to assign." + headId t = case t of T_NormalWord _ (x:_) -> getId x diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 4a01775..e559f62 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -2810,7 +2810,7 @@ readLetSuffix = many1 (readIoRedirect <|> try readLetExpression <|> readCmdWord) startPos <- getPosition expression <- readStringForParser readCmdWord let (unQuoted, newPos) = kludgeAwayQuotes expression startPos - subParse newPos readArithmeticContents unQuoted + subParse newPos (readArithmeticContents <* eof) unQuoted kludgeAwayQuotes :: String -> SourcePos -> (String, SourcePos) kludgeAwayQuotes s p = From fed4a048bccb7eccf30b5a18cab8e567114c8672 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Fri, 13 Aug 2021 23:11:20 -0700 Subject: [PATCH 167/459] Suppress SC2167 when name is "_" (fixes #2298) --- src/ShellCheck/Analytics.hs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index a415ca2..a45b2eb 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -3069,6 +3069,7 @@ checkUncheckedCdPushdPopd params root = prop_checkLoopVariableReassignment1 = verify checkLoopVariableReassignment "for i in *; do for i in *.bar; do true; done; done" prop_checkLoopVariableReassignment2 = verify checkLoopVariableReassignment "for i in *; do for((i=0; i<3; i++)); do true; done; done" prop_checkLoopVariableReassignment3 = verifyNot checkLoopVariableReassignment "for i in *; do for j in *.bar; do true; done; done" +prop_checkLoopVariableReassignment4 = verifyNot checkLoopVariableReassignment "for _ in *; do for _ in *.bar; do true; done; done" checkLoopVariableReassignment params token = sequence_ $ case token of T_ForIn {} -> check @@ -3077,6 +3078,7 @@ checkLoopVariableReassignment params token = where check = do str <- loopVariable token + guard $ str /= "_" next <- find (\x -> loopVariable x == Just str) path return $ do warn (getId token) 2165 "This nested loop overrides the index variable of its parent." From bb0a571a1e9ebf6aa2f92347f52103115fa494b6 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 16 Aug 2021 20:56:51 -0700 Subject: [PATCH 168/459] Improve warnings for bad parameter expansion (fixes #2297) --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 79 ++++++++++++++++++++++++++++------- src/ShellCheck/AnalyzerLib.hs | 15 ++++++- 3 files changed, 80 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c99322..ca29f28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - SC2292: Suggest [[ over [ in Bash/Ksh scripts (optional) - SC2293/SC2294: Warn when calling `eval` with arrays - SC2295: Warn about "${x#$y}" treating $y as a pattern when not quoted +- SC2296-SC2301: Improved warnings for bad parameter expansions ### Fixed - SC2102 about repetitions in ranges no longer triggers on [[ -v arr[xx] ]] diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index a45b2eb..d670977 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -136,7 +136,7 @@ nodeChecks = [ ,checkValidCondOps ,checkGlobbedRegex ,checkTestRedirects - ,checkIndirectExpansion + ,checkBadParameterSubstitution ,checkPS1Assignments ,checkBackticks ,checkInexplicablyUnquoted @@ -1608,29 +1608,79 @@ checkBackticks params (T_Backticked id list) | not (null list) = (fixWith [replaceStart id params 1 "$(", replaceEnd id params 1 ")"]) checkBackticks _ _ = return () -prop_checkIndirectExpansion1 = verify checkIndirectExpansion "${foo$n}" -prop_checkIndirectExpansion2 = verifyNot checkIndirectExpansion "${foo//$n/lol}" -prop_checkIndirectExpansion3 = verify checkIndirectExpansion "${$#}" -prop_checkIndirectExpansion4 = verify checkIndirectExpansion "${var${n}_$((i%2))}" -prop_checkIndirectExpansion5 = verifyNot checkIndirectExpansion "${bar}" -checkIndirectExpansion _ (T_DollarBraced i _ (T_NormalWord _ contents)) - | isIndirection contents = - err i 2082 "To expand via indirection, use arrays, ${!name} or (for sh only) eval." + +prop_checkBadParameterSubstitution1 = verify checkBadParameterSubstitution "${foo$n}" +prop_checkBadParameterSubstitution2 = verifyNot checkBadParameterSubstitution "${foo//$n/lol}" +prop_checkBadParameterSubstitution3 = verify checkBadParameterSubstitution "${$#}" +prop_checkBadParameterSubstitution4 = verify checkBadParameterSubstitution "${var${n}_$((i%2))}" +prop_checkBadParameterSubstitution5 = verifyNot checkBadParameterSubstitution "${bar}" +prop_checkBadParameterSubstitution6 = verify checkBadParameterSubstitution "${\"bar\"}" +prop_checkBadParameterSubstitution7 = verify checkBadParameterSubstitution "${{var}" +prop_checkBadParameterSubstitution8 = verify checkBadParameterSubstitution "${$(x)//x/y}" +prop_checkBadParameterSubstitution9 = verifyNot checkBadParameterSubstitution "$# ${#} $! ${!} ${!#} ${#!}" +prop_checkBadParameterSubstitution10 = verify checkBadParameterSubstitution "${'foo'}" +prop_checkBadParameterSubstitution11 = verify checkBadParameterSubstitution "${${x%.*}##*/}" + +checkBadParameterSubstitution _ t = + case t of + (T_DollarBraced i _ (T_NormalWord _ contents@(first:_))) -> + if isIndirection contents + then err i 2082 "To expand via indirection, use arrays, ${!name} or (for sh only) eval." + else checkFirst first + _ -> return () + where + isIndirection vars = let list = mapMaybe isIndirectionPart vars in not (null list) && and list + isIndirectionPart t = - case t of T_DollarExpansion _ _ -> Just True - T_Backticked _ _ -> Just True - T_DollarBraced _ _ _ -> Just True - T_DollarArithmetic _ _ -> Just True + case t of T_DollarExpansion {} -> Just True + T_Backticked {} -> Just True + T_DollarBraced {} -> Just True + T_DollarArithmetic {} -> Just True T_Literal _ s -> if all isVariableChar s then Nothing else Just False _ -> Just False -checkIndirectExpansion _ _ = return () + checkFirst t = + case t of + T_Literal id (c:_) -> + if isVariableChar c || isSpecialVariableChar c + then return () + else err id 2296 $ "Parameter expansions can't start with " ++ e4m [c] ++ ". Double check syntax." + + T_ParamSubSpecialChar {} -> return () + + T_DoubleQuoted id [T_Literal _ s] | isVariable s -> + err id 2297 "Double quotes must be outside ${}: ${\"invalid\"} vs \"${valid}\"." + + T_DollarBraced id braces _ | isUnmodifiedParameterExpansion t -> + err id 2298 $ + (if braces then "${${x}}" else "${$x}") + ++ " is invalid. For expansion, use ${x}. For indirection, use arrays, ${!x} or (for sh) eval." + + T_DollarBraced {} -> + err (getId t) 2299 "Parameter expansions can't be nested. Use temporary variables." + + _ | isCommandSubstitution t -> + err (getId t) 2300 "Parameter expansion can't be applied to command substitutions. Use temporary variables." + + _ -> err (getId t) 2301 $ "Parameter expansion starts with unexpected " ++ name t ++ ". Double check syntax." + + isVariable str = + case str of + [c] -> isVariableStartChar c || isSpecialVariableChar c || isDigit c + x -> isVariableName x + + name t = + case t of + T_SingleQuoted {} -> "quotes" + T_DoubleQuoted {} -> "quotes" + _ -> "syntax" + prop_checkInexplicablyUnquoted1 = verify checkInexplicablyUnquoted "echo 'var='value';'" prop_checkInexplicablyUnquoted2 = verifyNot checkInexplicablyUnquoted "'foo'*" @@ -4434,5 +4484,6 @@ checkUnquotedParameterExpansionPattern params x = surroundWith (getId t) params "\"" + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 439b48f..9ae0160 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -816,6 +816,7 @@ isConfusedGlobRegex _ = False isVariableStartChar x = x == '_' || isAsciiLower x || isAsciiUpper x isVariableChar x = isVariableStartChar x || isDigit x +isSpecialVariableChar = (`elem` "*@#?-$!") variableNameRegex = mkRegex "[_a-zA-Z][_a-zA-Z0-9]*" prop_isVariableName1 = isVariableName "_fo123" @@ -861,7 +862,7 @@ getBracedReference s = fromMaybe s $ let name = takeWhile isVariableChar s guard . not $ null name return name - getSpecial (c:_) | c `elem` "*@#?-$!" = return [c] + getSpecial (c:_) | isSpecialVariableChar c = return [c] getSpecial _ = fail "empty or not special" nameExpansion ('!':next:rest) = do -- e.g. ${!foo*bar*} @@ -955,5 +956,17 @@ isBashLike params = Dash -> False Sh -> False +-- Returns whether a token is a parameter expansion without any modifiers. +-- True for $var ${var} $1 $# +-- False for ${#var} ${var[x]} ${var:-0} +isUnmodifiedParameterExpansion t = + case t of + T_DollarBraced _ False _ -> True + T_DollarBraced _ _ list -> + let str = concat $ oversimplify list + in getBracedReference str == str + _ -> False + + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) From 8c0bf8d41f4aa18112acde9c9e594b66c93a2bdd Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 17 Aug 2021 12:50:40 -0700 Subject: [PATCH 169/459] Warn about looping over array values and using them as keys --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 84 +++++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca29f28..38b8e49 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - SC2293/SC2294: Warn when calling `eval` with arrays - SC2295: Warn about "${x#$y}" treating $y as a pattern when not quoted - SC2296-SC2301: Improved warnings for bad parameter expansions +- SC2302/SC2303: Warn about loops over array values when using them as keys ### Fixed - SC2102 about repetitions in ranges no longer triggers on [[ -v arr[xx] ]] diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index d670977..b768a0f 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -65,6 +65,7 @@ treeChecks = [ ,checkArrayAssignmentIndices ,checkUseBeforeDefinition ,checkAliasUsedInSameParsingUnit + ,checkArrayValueUsedAsIndex ] runAnalytics :: AnalysisSpec -> [TokenComment] @@ -4484,6 +4485,89 @@ checkUnquotedParameterExpansionPattern params x = surroundWith (getId t) params "\"" +prop_checkArrayValueUsedAsIndex1 = verifyTree checkArrayValueUsedAsIndex "for i in ${arr[@]}; do echo ${arr[i]}; done" +prop_checkArrayValueUsedAsIndex2 = verifyTree checkArrayValueUsedAsIndex "for i in ${arr[@]}; do echo ${arr[$i]}; done" +prop_checkArrayValueUsedAsIndex3 = verifyTree checkArrayValueUsedAsIndex "for i in ${arr[@]}; do echo $((arr[i])); done" +prop_checkArrayValueUsedAsIndex4 = verifyTree checkArrayValueUsedAsIndex "for i in ${arr1[@]} ${arr2[@]}; do echo ${arr1[$i]}; done" +prop_checkArrayValueUsedAsIndex5 = verifyTree checkArrayValueUsedAsIndex "for i in ${arr1[@]} ${arr2[@]}; do echo ${arr2[$i]}; done" +prop_checkArrayValueUsedAsIndex7 = verifyNotTree checkArrayValueUsedAsIndex "for i in ${arr[@]}; do echo ${arr[K]}; done" +prop_checkArrayValueUsedAsIndex8 = verifyNotTree checkArrayValueUsedAsIndex "for i in ${arr[@]}; do i=42; echo ${arr[i]}; done" +prop_checkArrayValueUsedAsIndex9 = verifyNotTree checkArrayValueUsedAsIndex "for i in ${arr[@]}; do echo ${arr2[i]}; done" + +checkArrayValueUsedAsIndex params _ = + doVariableFlowAnalysis read write Map.empty (variableFlow params) + where + write loop@T_ForIn {} _ name (DataString (SourceFrom words)) = do + modify $ Map.insert name (loop, mapMaybe f words) + return [] + where + f x = do + name <- getArrayName x + return (x, name) + + write _ _ name _ = do + modify $ Map.delete name + return [] + + read _ t name = do + varMap <- get + return $ fromMaybe [] $ do + (loop, arrays) <- Map.lookup name varMap + (arrayRef, arrayName) <- getArrayIfUsedAsIndex name t + -- Is this one of the 'for' arrays? + (loopWord, _) <- find ((==arrayName) . snd) arrays + -- Are we still in this loop? + guard $ getId loop `elem` map getId (getPath parents t) + return [ + makeComment WarningC (getId loopWord) 2302 "This loops over values. To loop over keys, use \"${!array[@]}\".", + makeComment WarningC (getId arrayRef) 2303 $ (e4m name) ++ " is an array value, not a key. Use directly or loop over keys instead." + ] + + parents = parentMap params + + getArrayName :: Token -> Maybe String + getArrayName t = do + [T_DollarBraced _ _ l] <- return $ getWordParts t + let str = concat $ oversimplify l + guard $ getBracedModifier str == "[@]" && not ("!" `isPrefixOf` str) + return $ getBracedReference str + + -- This is much uglier than it should be + getArrayIfUsedAsIndex :: String -> Token -> Maybe (Token, String) + getArrayIfUsedAsIndex name t = + case t of + T_DollarBraced _ _ list -> do + let ref = getBracedReference $ concat $ oversimplify list + guard $ ref == name + -- We found a $name. Look up the chain to see if it's ${arr[$name]} + list@T_NormalWord {} <- Map.lookup (getId t) parents + (T_DollarBraced _ _ parentList) <- Map.lookup (getId list) parents + (T_Literal _ head : index : T_Literal _ tail : _) <- return $ getWordParts parentList + let str = concat $ oversimplify list + let modifier = getBracedModifier str + guard $ getId index == getId t + guard $ "[${VAR}]" `isPrefixOf` modifier + return (t, getBracedReference str) + + T_NormalWord wordId list -> do + -- We found just name. Check if it's part of ${something[name]} + parent@(T_DollarBraced _ _ parentList) <- Map.lookup wordId parents + let str = concat $ oversimplify t + let modifier = getBracedModifier str + guard $ ("[" ++ name ++ "]") `isPrefixOf` modifier + return (parent, getBracedReference str) + + TA_Variable indexId ref [] -> do + -- We found arithmetic name. See if it's part of arithmetic arr[name] + guard $ ref == name + (TA_Sequence seqId [element]) <- Map.lookup indexId parents + guard $ getId element == indexId + parent@(TA_Variable arrayId arrayName [element]) <- Map.lookup seqId parents + guard $ getId element == seqId + return (parent, arrayName) + + _ -> Nothing + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) From c61fc7546e4660b117a89b2a3904c7317a1d694a Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 17 Aug 2021 14:14:05 -0700 Subject: [PATCH 170/459] Don't warn about variables guarded with :+ (fixes #2296) --- src/ShellCheck/Analytics.hs | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index b768a0f..9aff3ab 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2379,6 +2379,8 @@ prop_checkUnassignedReferences_minusNBraced = verifyNotTree checkUnassignedRefe prop_checkUnassignedReferences_minusZBraced = verifyNotTree checkUnassignedReferences "if [ -z \"${x}\" ]; then echo \"\"; fi" prop_checkUnassignedReferences_minusNDefault = verifyNotTree checkUnassignedReferences "if [ -n \"${x:-}\" ]; then echo $x; fi" prop_checkUnassignedReferences_minusZDefault = verifyNotTree checkUnassignedReferences "if [ -z \"${x:-}\" ]; then echo \"\"; fi" +prop_checkUnassignedReferences50 = verifyNotTree checkUnassignedReferences "echo ${foo:+bar}" +prop_checkUnassignedReferences51 = verifyNotTree checkUnassignedReferences "echo ${foo:+$foo}" checkUnassignedReferences = checkUnassignedReferences' False checkUnassignedReferences' includeGlobals params t = warnings @@ -2411,8 +2413,8 @@ checkUnassignedReferences' includeGlobals params t = warnings warningForGlobals var place = do match <- getBestMatch var - return $ warn (getId place) 2153 $ - "Possible misspelling: " ++ var ++ " may not be assigned, but " ++ match ++ " is." + return $ info (getId place) 2153 $ + "Possible misspelling: " ++ var ++ " may not be assigned. Did you mean " ++ match ++ "?" warningForLocals var place = return $ warn (getId place) 2154 $ @@ -2427,7 +2429,7 @@ checkUnassignedReferences' includeGlobals params t = warnings warningFor (var, place) = do guard $ isVariableName var - guard . not $ isInArray var place || isGuarded place + guard . not $ isException var place || isGuarded place (if includeGlobals || isLocal var then warningForLocals else warningForGlobals) var place @@ -2436,11 +2438,22 @@ checkUnassignedReferences' includeGlobals params t = warnings -- Due to parsing, foo=( [bar]=baz ) parses 'bar' as a reference even for assoc arrays. -- Similarly, ${foo[bar baz]} may not be referencing bar/baz. Just skip these. - isInArray var t = any isArray $ getPath (parentMap params) t + -- We can also have ${foo:+$foo} should be treated like [[ -n $foo ]] && echo $foo + isException var t = any shouldExclude $ getPath (parentMap params) t where - isArray T_Array {} = True - isArray (T_DollarBraced _ _ l) | var /= getBracedReference (concat $ oversimplify l) = True - isArray _ = False + shouldExclude t = + case t of + T_Array {} -> True + (T_DollarBraced _ _ l) -> + let str = concat $ oversimplify l + ref = getBracedReference str + mod = getBracedModifier str + in + -- Either we're used as an array index like ${arr[here]} + ref /= var || + -- or the reference is guarded by a parent, ${here:+foo$here} + "+" `isPrefixOf` mod || ":+" `isPrefixOf` mod + _ -> False isGuarded (T_DollarBraced _ _ v) = rest `matches` guardRegex From da7b28213ef65ef4a0d702831d1d358f37e4f114 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 17 Aug 2021 21:53:27 -0700 Subject: [PATCH 171/459] Recognize wait -p as assigning a variable (fixes #2179) --- src/ShellCheck/ASTLib.hs | 33 +++++++++++++++++++++++++++++++ src/ShellCheck/Analytics.hs | 2 ++ src/ShellCheck/AnalyzerLib.hs | 19 +++++++++--------- src/ShellCheck/Checks/Commands.hs | 12 +++++++++++ 4 files changed, 57 insertions(+), 9 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 019fc3c..7b5efdd 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -228,6 +228,39 @@ getOpts (gnu, arbitraryLongOpts) string longopts args = process args listToArgs = map (\x -> ("", (x, x))) + +-- Generic getOpts that doesn't rely on a format string, but may also be inaccurate. +-- This provides a best guess interpretation instead of failing when new options are added. +-- +-- "--" is treated as end of arguments +-- "--anything[=foo]" is treated as a long option without argument +-- "-any" is treated as -a -n -y, with the next arg as an option to -y unless it starts with - +-- anything else is an argument +getGenericOpts :: [Token] -> [(String, (Token, Token))] +getGenericOpts = process + where + process (token:rest) = + case getLiteralStringDef "\0" token of + "--" -> map (\c -> ("", (c,c))) rest + '-':'-':word -> (takeWhile (`notElem` "\0=") word, (token, token)) : process rest + '-':optString -> + let opts = takeWhile (/= '\0') optString + in + case rest of + next:_ | "-" `isPrefixOf` getLiteralStringDef "\0" next -> + map (\c -> ([c], (token, token))) opts ++ process rest + next:remainder -> + case reverse opts of + last:initial -> + map (\c -> ([c], (token, token))) (reverse initial) + ++ [([last], (token, next))] + ++ process remainder + [] -> process remainder + [] -> map (\c -> ([c], (token, token))) opts + _ -> ("", (token, token)) : process rest + process [] = [] + + -- Is this an expansion of multiple items of an array? isArrayExpansion (T_DollarBraced _ _ l) = let string = concat $ oversimplify l in diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 9aff3ab..e1e55fd 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1968,6 +1968,7 @@ prop_checkSpacefulness41= verifyNotTree checkSpacefulness "exec $1 --flags" prop_checkSpacefulness42= verifyNotTree checkSpacefulness "run $1 --flags" prop_checkSpacefulness43= verifyNotTree checkSpacefulness "$foo=42" prop_checkSpacefulness44= verifyTree checkSpacefulness "#!/bin/sh\nexport var=$value" +prop_checkSpacefulness45= verifyNotTree checkSpacefulness "wait -zzx -p foo; echo $foo" data SpaceStatus = SpaceSome | SpaceNone | SpaceEmpty deriving (Eq) instance Semigroup SpaceStatus where @@ -2381,6 +2382,7 @@ prop_checkUnassignedReferences_minusNDefault = verifyNotTree checkUnassignedRefe prop_checkUnassignedReferences_minusZDefault = verifyNotTree checkUnassignedReferences "if [ -z \"${x:-}\" ]; then echo \"\"; fi" prop_checkUnassignedReferences50 = verifyNotTree checkUnassignedReferences "echo ${foo:+bar}" prop_checkUnassignedReferences51 = verifyNotTree checkUnassignedReferences "echo ${foo:+$foo}" +prop_checkUnassignedReferences52 = verifyNotTree checkUnassignedReferences "wait -p pid; echo $pid" checkUnassignedReferences = checkUnassignedReferences' False checkUnassignedReferences' includeGlobals params t = warnings diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 9ae0160..633543a 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -617,6 +617,7 @@ getModifiedVariableCommand base@(T_SimpleCommand id cmdPrefix (T_NormalWord _ (T return (base, base, "@", DataString $ SourceFrom params) "printf" -> maybeToList $ getPrintfVariable rest + "wait" -> maybeToList $ getWaitVariable rest "mapfile" -> maybeToList $ getMapfileArray base rest "readarray" -> maybeToList $ getMapfileArray base rest @@ -674,15 +675,15 @@ getModifiedVariableCommand base@(T_SimpleCommand id cmdPrefix (T_NormalWord _ (T _ -> return (t:fromMaybe [] (getSetParams rest)) getSetParams [] = Nothing - getPrintfVariable list = f $ map (\x -> (x, getLiteralString x)) list - where - f ((_, Just "-v") : (t, Just var) : _) = return (base, t, varName, varType $ SourceFrom list) - where - (varName, varType) = case elemIndex '[' var of - Just i -> (take i var, DataArray) - Nothing -> (var, DataString) - f (_:rest) = f rest - f [] = fail "not found" + getPrintfVariable list = getFlagAssignedVariable "v" (SourceFrom list) $ getBsdOpts "v:" list + getWaitVariable list = getFlagAssignedVariable "p" SourceInteger $ return $ getGenericOpts list + + getFlagAssignedVariable str dataSource maybeFlags = do + flags <- maybeFlags + (_, (flag, value)) <- find ((== str) . fst) flags + variableName <- getLiteralStringExt (const $ return "!") value + let (baseName, index) = span (/= '[') variableName + return (base, value, baseName, (if null index then DataString else DataArray) dataSource) -- mapfile has some curious syntax allowing flags plus 0..n variable names -- where only the first non-option one is used if any. diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index c0cd9a3..6c7bdc5 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -138,18 +138,30 @@ prop_checkGetOptsS3 = checkGetOpts "-f -x" ["f", "x"] [] $ getOpts (True, True) prop_checkGetOptsS4 = checkGetOpts "-f -x" ["f"] [] $ getOpts (True, True) "f:" [] prop_checkGetOptsS5 = checkGetOpts "-fx" [] [] $ getOpts (True, True) "fx:" [] +prop_checkGenericOptsS1 = checkGetOpts "-f x" ["f"] [] $ return . getGenericOpts +prop_checkGenericOptsS2 = checkGetOpts "-abc x" ["a", "b", "c"] [] $ return . getGenericOpts +prop_checkGenericOptsS3 = checkGetOpts "-abc -x" ["a", "b", "c", "x"] [] $ return . getGenericOpts +prop_checkGenericOptsS4 = checkGetOpts "-x" ["x"] [] $ return . getGenericOpts + -- Long options prop_checkGetOptsL1 = checkGetOpts "--foo=bar baz" ["foo"] ["baz"] $ getOpts (True, False) "" [("foo", True)] prop_checkGetOptsL2 = checkGetOpts "--foo bar baz" ["foo"] ["baz"] $ getOpts (True, False) "" [("foo", True)] prop_checkGetOptsL3 = checkGetOpts "--foo baz" ["foo"] ["baz"] $ getOpts (True, True) "" [] prop_checkGetOptsL4 = checkGetOpts "--foo baz" [] [] $ getOpts (True, False) "" [] +prop_checkGenericOptsL1 = checkGetOpts "--foo=bar" ["foo"] [] $ return . getGenericOpts +prop_checkGenericOptsL2 = checkGetOpts "--foo bar" ["foo"] ["bar"] $ return . getGenericOpts +prop_checkGenericOptsL3 = checkGetOpts "-x --foo" ["x", "foo"] [] $ return . getGenericOpts + -- Know when to terminate prop_checkGetOptsT1 = checkGetOpts "-a x -b" ["a", "b"] ["x"] $ getOpts (True, True) "ab" [] prop_checkGetOptsT2 = checkGetOpts "-a x -b" ["a"] ["x","-b"] $ getOpts (False, True) "ab" [] prop_checkGetOptsT3 = checkGetOpts "-a -- -b" ["a"] ["-b"] $ getOpts (True, True) "ab" [] prop_checkGetOptsT4 = checkGetOpts "-a -- -b" ["a", "b"] [] $ getOpts (True, True) "a:b" [] +prop_checkGenericOptsT1 = checkGetOpts "-x -- -y" ["x"] ["-y"] $ return . getGenericOpts +prop_checkGenericOptsT2 = checkGetOpts "-xy --" ["x", "y"] [] $ return . getGenericOpts + buildCommandMap :: [CommandCheck] -> Map.Map CommandName (Token -> Analysis) buildCommandMap = foldl' addCheck Map.empty From 5b6fd60279a70539c27117f70e5386b984d3d9c3 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 22 Aug 2021 11:55:01 -0700 Subject: [PATCH 172/459] Improve warnings for expr (fixes #2033) --- CHANGELOG.md | 3 ++ src/ShellCheck/AnalyzerLib.hs | 2 +- src/ShellCheck/Checks/Commands.hs | 71 +++++++++++++++++++++++++++---- 3 files changed, 67 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 38b8e49..4b510ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ - SC2295: Warn about "${x#$y}" treating $y as a pattern when not quoted - SC2296-SC2301: Improved warnings for bad parameter expansions - SC2302/SC2303: Warn about loops over array values when using them as keys +- SC2304-SC2306: Warn about unquoted globs in expr arguments +- SC2307: Warn about insufficient number of arguments to expr +- SC2308: Suggest other approaches for non-standard expr extensions ### Fixed - SC2102 about repetitions in ranges no longer triggers on [[ -v arr[xx] ]] diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 633543a..662eff5 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -142,7 +142,7 @@ producesComments c s = do prRoot pr let spec = defaultSpec pr let params = makeParameters spec - return . not . null $ runChecker params c + return . not . null $ filterByAnnotation spec params $ runChecker params c makeComment :: Severity -> Id -> Code -> String -> TokenComment makeComment severity id code note = diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 6c7bdc5..76ac9a7 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -57,7 +57,7 @@ commandChecks :: [CommandCheck] commandChecks = [ checkTr ,checkFindNameGlob - ,checkNeedlessExpr + ,checkExpr ,checkGrepRe ,checkTrapQuotes ,checkReturn @@ -254,19 +254,74 @@ checkFindNameGlob = CommandCheck (Basename "find") (f . arguments) where acc b -prop_checkNeedlessExpr = verify checkNeedlessExpr "foo=$(expr 3 + 2)" -prop_checkNeedlessExpr2 = verify checkNeedlessExpr "foo=`echo \\`expr 3 + 2\\``" -prop_checkNeedlessExpr3 = verifyNot checkNeedlessExpr "foo=$(expr foo : regex)" -prop_checkNeedlessExpr4 = verifyNot checkNeedlessExpr "foo=$(expr foo \\< regex)" -checkNeedlessExpr = CommandCheck (Basename "expr") f where - f t = +prop_checkExpr = verify checkExpr "foo=$(expr 3 + 2)" +prop_checkExpr2 = verify checkExpr "foo=`echo \\`expr 3 + 2\\``" +prop_checkExpr3 = verifyNot checkExpr "foo=$(expr foo : regex)" +prop_checkExpr4 = verifyNot checkExpr "foo=$(expr foo \\< regex)" +prop_checkExpr5 = verify checkExpr "# shellcheck disable=SC2003\nexpr match foo bar" +prop_checkExpr6 = verify checkExpr "# shellcheck disable=SC2003\nexpr foo : fo*" +prop_checkExpr7 = verify checkExpr "# shellcheck disable=SC2003\nexpr 5 -3" +prop_checkExpr8 = verifyNot checkExpr "# shellcheck disable=SC2003\nexpr \"$@\"" +prop_checkExpr9 = verifyNot checkExpr "# shellcheck disable=SC2003\nexpr 5 $rest" +prop_checkExpr10 = verify checkExpr "# shellcheck disable=SC2003\nexpr length \"$var\"" +prop_checkExpr11 = verify checkExpr "# shellcheck disable=SC2003\nexpr foo > bar" +prop_checkExpr12 = verify checkExpr "# shellcheck disable=SC2003\nexpr 1 | 2" +prop_checkExpr13 = verify checkExpr "# shellcheck disable=SC2003\nexpr 1 * 2" +prop_checkExpr14 = verify checkExpr "# shellcheck disable=SC2003\nexpr \"$x\" >= \"$y\"" + +checkExpr = CommandCheck (Basename "expr") f where + f t = do when (all (`notElem` exceptions) (words $ arguments t)) $ style (getId $ getCommandTokenOrThis t) 2003 "expr is antiquated. Consider rewriting this using $((..)), ${} or [[ ]]." + + case arguments t of + [lhs, op, rhs] -> do + checkOp lhs + case getWordParts op of + [T_Glob _ "*"] -> + err (getId op) 2304 + "* must be escaped to multiply: \\*. Modern $((x * y)) avoids this issue." + [T_Literal _ ":"] | isGlob rhs -> + warn (getId rhs) 2305 + "Quote regex argument to expr to avoid it expanding as a glob." + _ -> return () + + [single] | not (willSplit single) -> + warn (getId single) 2307 + "'expr' expects 3+ arguments but sees 1. Make sure each operator/operand is a separate argument, and escape <>&|." + + [first, second] | + (fromMaybe "" $ getLiteralString first) /= "length" + && not (willSplit first || willSplit second) -> do + checkOp first + warn (getId t) 2307 + "'expr' expects 3+ arguments, but sees 2. Make sure each operator/operand is a separate argument, and escape <>&|." + + (first:rest) -> do + checkOp first + forM_ rest $ \t -> + -- We already find 95%+ of multiplication and regex earlier, so don't bother classifying this further. + when (isGlob t) $ warn (getId t) 2306 "Escape glob characters in arguments to expr to avoid pathname expansion." + + _ -> return () + -- These operators are hard to replicate in POSIX - exceptions = [ ":", "<", ">", "<=", ">=" ] + exceptions = [ ":", "<", ">", "<=", ">=", + -- We can offer better suggestions for these + "match", "length", "substr", "index"] words = mapMaybe getLiteralString + checkOp side = + case getLiteralString side of + Just "match" -> msg "'expr match' has unspecified results. Prefer 'expr str : regex'." + Just "length" -> msg "'expr length' has unspecified results. Prefer ${#var}." + Just "substr" -> msg "'expr substr' has unspecified results. Prefer 'cut' or ${var#???}." + Just "index" -> msg "'expr index' has unspecified results. Prefer x=${var%%[chars]*}; $((${#x}+1))." + _ -> return () + where + msg = info (getId side) 2308 + prop_checkGrepRe1 = verify checkGrepRe "cat foo | grep *.mp3" prop_checkGrepRe2 = verify checkGrepRe "grep -Ev cow*test *.mp3" From 7384cec3f6f4e29212c58e7d654e7853d4d173fe Mon Sep 17 00:00:00 2001 From: a1346054 <36859588+a1346054@users.noreply.github.com> Date: Wed, 25 Aug 2021 14:15:36 +0000 Subject: [PATCH 173/459] Fix redirect in LICENSE file The file was obtained from: https://www.gnu.org/licenses/gpl-3.0.txt --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 0507f1f..0df6056 100644 --- a/LICENSE +++ b/LICENSE @@ -681,4 +681,4 @@ into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Lesser General Public License instead of this License. But first, please read -. +. From 98c7934c46ec665bd1e2565cf3cb3f8b6c34e2ef Mon Sep 17 00:00:00 2001 From: a1346054 <36859588+a1346054@users.noreply.github.com> Date: Wed, 25 Aug 2021 16:17:56 +0000 Subject: [PATCH 174/459] Remove trailing whitespace --- .github_deploy | 1 - CHANGELOG.md | 6 +++--- snap/snapcraft.yaml | 4 ++-- striptests | 1 - 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.github_deploy b/.github_deploy index 3de0ac2..82c8ec5 100755 --- a/.github_deploy +++ b/.github_deploy @@ -26,4 +26,3 @@ do done gh release upload "$tag" "${files[@]}" --clobber || exit 1 done - diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b510ab..f479c95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -222,7 +222,7 @@ - SC2185: Suggest explicitly adding path for `find` - SC2184: Warn about unsetting globs (e.g. `unset foo[1]`) - SC2183: Warn about `printf` with more formatters than variables -- SC2182: Warn about ignored arguments with `printf` +- SC2182: Warn about ignored arguments with `printf` - SC2181: Suggest using command directly instead of `if [ $? -eq 0 ]` - SC1106: Warn when using `test` operators in `(( 1 -eq 2 ))` @@ -393,7 +393,7 @@ ### Added - SC2121: Warn about trying to `set` variables, e.g. `set var = value` - SC2120/SC2119: Warn when a function uses `$1..` if none are ever passed -- SC2117: Warn when using `su` in interactive mode, e.g. `su foo; whoami` +- SC2117: Warn when using `su` in interactive mode, e.g. `su foo; whoami` - SC2116: Detect useless use of echo, e.g. `for i in $(echo $var)` - SC2115/SC2114: Detect some catastrophic `rm -r "$empty/"` mistakes - SC1081: Warn when capitalizing keywords like `While` @@ -444,7 +444,7 @@ ### Removed - Suggestions about using parameter expansion over basename -- The `jsoncheck` binary. Use `shellcheck -f json` instead. +- The `jsoncheck` binary. Use `shellcheck -f json` instead. ## v0.2.0 - 2013-10-27 diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 2dc4831..e14b854 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -16,12 +16,12 @@ description: | advanced user's otherwise working script to fail under future circumstances. - By default ShellCheck can only check non-hidden files under /home, to make + By default ShellCheck can only check non-hidden files under /home, to make ShellCheck be able to check files under /media and /run/media you must connect it to the `removable-media` interface manually: # snap connect shellcheck:removable-media - + version: git base: core18 grade: stable diff --git a/striptests b/striptests index 8281f78..c68c16a 100755 --- a/striptests +++ b/striptests @@ -75,4 +75,3 @@ find . -name '.git' -prune -o -type f -name '*.hs' -print | do modify "$file" detestify done - From c85ce2cb06cce93cccea86b540528ae9e5dde153 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 26 Aug 2021 18:50:40 -0700 Subject: [PATCH 175/459] Add `rg` to list of commands ignored for SC2016 (fixes #2209) --- src/ShellCheck/Analytics.hs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index e1e55fd..97863dd 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1040,6 +1040,7 @@ checkSingleQuotedVariables params t@(T_SingleQuoted id s) = ,"dpkg-query" ,"jq" -- could also check that user provides --arg ,"rename" + ,"rg" ,"unset" ,"git filter-branch" ,"mumps -run %XCMD" From 81b7ee55980962a4631aef5bf98b3cc21822c5a4 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 26 Aug 2021 19:40:21 -0700 Subject: [PATCH 176/459] Don't warn about unused variables starting with _ (fixes #1498) --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b510ab..03c8395 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ - SC2048: Warning about $\* now also applies to ${array[\*]} - SC2181 now only triggers on single condition tests like `[ $? = 0 ]`. - Quote warnings are now emitted for declaration utilities in sh +- Leading `_` can now be used to suppress warnings about unused variables ## v0.7.2 - 2021-04-19 diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 97863dd..8e0589c 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2310,6 +2310,7 @@ prop_checkUnused44= verifyNotTree checkUnusedAssignments "DEFINE_string \"foo$ib prop_checkUnused45= verifyTree checkUnusedAssignments "readonly foo=bar" prop_checkUnused46= verifyTree checkUnusedAssignments "readonly foo=(bar)" prop_checkUnused47= verifyNotTree checkUnusedAssignments "a=1; alias hello='echo $a'" +prop_checkUnused48= verifyNotTree checkUnusedAssignments "_a=1" checkUnusedAssignments params t = execWriter (mapM_ warnFor unused) where flow = variableFlow params @@ -2326,8 +2327,9 @@ checkUnusedAssignments params t = execWriter (mapM_ warnFor unused) unused = Map.assocs $ Map.difference assignments references warnFor (name, token) = - warn (getId token) 2034 $ - name ++ " appears unused. Verify use (or export if used externally)." + unless ("_" `isPrefixOf` name) $ + warn (getId token) 2034 $ + name ++ " appears unused. Verify use (or export if used externally)." stripSuffix = takeWhile isVariableChar defaultMap = Map.fromList $ zip internalVariables $ repeat () From 081f7eba24f9d0e908a12cde540dcb181be7d679 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 26 Aug 2021 23:05:14 -0700 Subject: [PATCH 177/459] Fix parsing of [$var] (fixes #2309) --- src/ShellCheck/ASTLib.hs | 26 ++++++++++++++++++++++---- src/ShellCheck/Analytics.hs | 2 ++ src/ShellCheck/AnalyzerLib.hs | 29 ++++++++++++++++++----------- src/ShellCheck/Checks/Commands.hs | 3 +++ src/ShellCheck/Parser.hs | 13 +++++++++---- 5 files changed, 54 insertions(+), 19 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 7b5efdd..20e5be4 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -59,10 +59,28 @@ willSplit x = T_NormalWord _ l -> any willSplit l _ -> False -isGlob T_Extglob {} = True -isGlob T_Glob {} = True -isGlob (T_NormalWord _ l) = any isGlob l -isGlob _ = False +isGlob t = case t of + T_Extglob {} -> True + T_Glob {} -> True + T_NormalWord _ l -> any isGlob l || hasSplitRange l + _ -> False + where + -- foo[x${var}y] gets parsed as foo,[,x,$var,y], + -- so check if there's such an interval + hasSplitRange l = + let afterBracket = dropWhile (not . isHalfOpenRange) l + in any isClosingRange afterBracket + + isHalfOpenRange t = + case t of + T_Literal _ "[" -> True + _ -> False + + isClosingRange t = + case t of + T_Literal _ str -> ']' `elem` str + _ -> False + -- Is this shell word a constant? isConstant token = diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 8e0589c..f35fc0d 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2311,6 +2311,8 @@ prop_checkUnused45= verifyTree checkUnusedAssignments "readonly foo=bar" prop_checkUnused46= verifyTree checkUnusedAssignments "readonly foo=(bar)" prop_checkUnused47= verifyNotTree checkUnusedAssignments "a=1; alias hello='echo $a'" prop_checkUnused48= verifyNotTree checkUnusedAssignments "_a=1" +prop_checkUnused49= verifyNotTree checkUnusedAssignments "declare -A array; key=a; [[ -v array[$key] ]]" + checkUnusedAssignments params t = execWriter (mapM_ warnFor unused) where flow = variableFlow params diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 662eff5..5b389d7 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -497,14 +497,8 @@ getModifiedVariables t = -- Count [[ -v foo ]] as an "assignment". -- This is to prevent [ -v foo ] being unassigned or unused. - TC_Unary id _ "-v" token -> do - str <- fmap (takeWhile (/= '[')) $ -- Quoted index - flip getLiteralStringExt token $ \x -> - case x of - T_Glob _ s -> return s -- Unquoted index - _ -> [] - - guard . not . null $ str + TC_Unary id _ "-v" token -> maybeToList $ do + str <- getVariableForTestDashV token return (t, token, str, DataString SourceChecked) TC_Unary _ _ "-n" token -> markAsChecked t token @@ -724,6 +718,20 @@ getIndexReferences s = fromMaybe [] $ do where re = mkRegex "(\\[.*\\])" +-- Given a NormalWord like foo or foo[$bar], get foo. +-- Primarily used to get references for [[ -v foo[bar] ]] +getVariableForTestDashV :: Token -> Maybe String +getVariableForTestDashV t = do + str <- takeWhile ('[' /=) <$> getLiteralStringExt toStr t + guard $ isVariableName str + return str + where + -- foo[bar] gets parsed with [bar] as a glob, so undo that + toStr (T_Glob _ s) = return s + -- Turn foo[$x] into foo[\0] so that we can get the constant array name + -- in a non-constant expression (while filtering out foo$x[$y]) + toStr _ = return "\0" + prop_getOffsetReferences1 = getOffsetReferences ":bar" == ["bar"] prop_getOffsetReferences2 = getOffsetReferences ":bar:baz" == ["bar", "baz"] prop_getOffsetReferences3 = getOffsetReferences "[foo]:bar" == ["bar"] @@ -782,9 +790,8 @@ getReferencedVariables parents t = T_Glob _ s -> return s -- Also when parsed as globs _ -> [] - getIfReference context token = do - str@(h:_) <- getLiteralStringExt literalizer token - when (isDigit h) $ fail "is a number" + getIfReference context token = maybeToList $ do + str <- getVariableForTestDashV token return (context, token, getBracedReference str) isArithmeticAssignment t = case getPath parents t of diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 76ac9a7..d0ada73 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -845,6 +845,9 @@ checkAliasesExpandEarly = CommandCheck (Exactly "alias") (f . arguments) prop_checkUnsetGlobs1 = verify checkUnsetGlobs "unset foo[1]" prop_checkUnsetGlobs2 = verifyNot checkUnsetGlobs "unset foo" +prop_checkUnsetGlobs3 = verify checkUnsetGlobs "unset foo[$i]" +prop_checkUnsetGlobs4 = verify checkUnsetGlobs "unset foo[x${i}y]" +prop_checkUnsetGlobs5 = verifyNot checkUnsetGlobs "unset foo][" checkUnsetGlobs = CommandCheck (Exactly "unset") (mapM_ check . arguments) where check arg = diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index e559f62..45434a8 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -1372,6 +1372,8 @@ prop_readGlob5 = isOk readGlob "[^[:alpha:]1-9]" prop_readGlob6 = isOk readGlob "[\\|]" prop_readGlob7 = isOk readGlob "[^[]" prop_readGlob8 = isOk readGlob "[*?]" +prop_readGlob9 = isOk readGlob "[!]^]" +prop_readGlob10 = isOk readGlob "[]]" readGlob = readExtglob <|> readSimple <|> readClass <|> readGlobbyLiteral where readSimple = do @@ -1379,22 +1381,25 @@ readGlob = readExtglob <|> readSimple <|> readClass <|> readGlobbyLiteral c <- oneOf "*?" id <- endSpan start return $ T_Glob id [c] - -- Doesn't handle weird things like [^]a] and [$foo]. fixme? readClass = try $ do start <- startSpan char '[' - s <- many1 (predefined <|> readNormalLiteralPart "]" <|> globchars) + negation <- charToString (oneOf "!^") <|> return "" + leadingBracket <- charToString (oneOf "]") <|> return "" + s <- many (predefined <|> readNormalLiteralPart "]" <|> globchars) + guard $ not (null leadingBracket) || not (null s) char ']' id <- endSpan start - return $ T_Glob id $ "[" ++ concat s ++ "]" + return $ T_Glob id $ "[" ++ concat (negation:leadingBracket:s) ++ "]" where - globchars = fmap return . oneOf $ "!$[" ++ extglobStartChars + globchars = charToString $ oneOf $ "![" ++ extglobStartChars predefined = do try $ string "[:" s <- many1 letter string ":]" return $ "[:" ++ s ++ ":]" + charToString = fmap return readGlobbyLiteral = do start <- startSpan c <- extglobStart <|> char '[' From 9d64d78c320cfcd89e46d1de53f6c6d8e9487b4f Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 28 Aug 2021 21:19:45 -0700 Subject: [PATCH 178/459] Allow running this repo as a pre-commit hook --- .dockerignore | 9 +++------ .pre-commit-hooks.yaml | 6 ++++++ Dockerfile | 18 ++++++++++++++++++ test/distrotest | 30 ++++++++++++++++++++++++++++++ 4 files changed, 57 insertions(+), 6 deletions(-) create mode 100644 .pre-commit-hooks.yaml create mode 100644 Dockerfile diff --git a/.dockerignore b/.dockerignore index 39d8893..49f90c9 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,3 @@ -* -!LICENSE -!Setup.hs -!ShellCheck.cabal -!shellcheck.hs -!src +dist +dist-newstyle +.git diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml new file mode 100644 index 0000000..9c5b4bc --- /dev/null +++ b/.pre-commit-hooks.yaml @@ -0,0 +1,6 @@ +- id: shellcheck + name: shellcheck + description: Static analysis tool for shell scripts + types: [shell] + language: docker + entry: shellcheck diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..0f8e07c --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +# This file builds ShellCheck for pre-commit. +# +# It may also be useful for local development, but it is notably NOT +# used to build the official ShellCheck docker images. + +FROM ubuntu:20.04 +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y cabal-install +RUN cabal update + +# Install dependencies separately for more efficient iteration +COPY ShellCheck.cabal /build/ +RUN cd /build && cabal install --dependencies-only + +# Now build the rest +COPY . /build/ +RUN cd /build && cabal build shellcheck +RUN find /build -type f -name shellcheck -perm /111 -exec cp -f {} /bin \; +ENTRYPOINT ["/bin/shellcheck"] diff --git a/test/distrotest b/test/distrotest index 346a706..8aaa84d 100755 --- a/test/distrotest +++ b/test/distrotest @@ -1,6 +1,9 @@ #!/bin/bash # This script runs 'buildtest' on each of several distros # via Docker. + +# shellcheck disable=SC2016 + set -o pipefail exec 3>&1 4>&2 @@ -32,6 +35,33 @@ echo "Logging to $log" >&3 exec >> "$log" 2>&1 final=0 + +echo "Trying to build pre-commit docker image" +if ! docker build --tag precommit . +then + final=1 + echo "pre-commit image failed to build" +else + if printf '%s\n' '#!/bin/sh' 'echo $1' | docker run -i precommit - + then + final=1 + echo "pre-commit image succeeds with incorrect example" + fi + + if ! printf '%s\n' '#!/bin/sh' 'echo "$1"' | docker run -i precommit - + then + final=1 + echo "pre-commit image fails with correct example" + fi +fi + +if [[ $final -ne 0 ]] +then + echo >&3 "pre-commit image failure, see log" +else + echo >&3 "pre-commit image succeeded" +fi + while read -r distro setup do [[ "$distro" = "#"* || -z "$distro" ]] && continue From b0f05018c1acc96b76b5e4ac805ed86fc47dd3ae Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 29 Aug 2021 12:12:08 -0700 Subject: [PATCH 179/459] Revert "Allow running this repo as a pre-commit hook" This reverts commit 9d64d78c320cfcd89e46d1de53f6c6d8e9487b4f. --- .dockerignore | 9 ++++++--- .pre-commit-hooks.yaml | 6 ------ Dockerfile | 18 ------------------ test/distrotest | 30 ------------------------------ 4 files changed, 6 insertions(+), 57 deletions(-) delete mode 100644 .pre-commit-hooks.yaml delete mode 100644 Dockerfile diff --git a/.dockerignore b/.dockerignore index 49f90c9..39d8893 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,6 @@ -dist -dist-newstyle -.git +* +!LICENSE +!Setup.hs +!ShellCheck.cabal +!shellcheck.hs +!src diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml deleted file mode 100644 index 9c5b4bc..0000000 --- a/.pre-commit-hooks.yaml +++ /dev/null @@ -1,6 +0,0 @@ -- id: shellcheck - name: shellcheck - description: Static analysis tool for shell scripts - types: [shell] - language: docker - entry: shellcheck diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 0f8e07c..0000000 --- a/Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -# This file builds ShellCheck for pre-commit. -# -# It may also be useful for local development, but it is notably NOT -# used to build the official ShellCheck docker images. - -FROM ubuntu:20.04 -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y cabal-install -RUN cabal update - -# Install dependencies separately for more efficient iteration -COPY ShellCheck.cabal /build/ -RUN cd /build && cabal install --dependencies-only - -# Now build the rest -COPY . /build/ -RUN cd /build && cabal build shellcheck -RUN find /build -type f -name shellcheck -perm /111 -exec cp -f {} /bin \; -ENTRYPOINT ["/bin/shellcheck"] diff --git a/test/distrotest b/test/distrotest index 8aaa84d..346a706 100755 --- a/test/distrotest +++ b/test/distrotest @@ -1,9 +1,6 @@ #!/bin/bash # This script runs 'buildtest' on each of several distros # via Docker. - -# shellcheck disable=SC2016 - set -o pipefail exec 3>&1 4>&2 @@ -35,33 +32,6 @@ echo "Logging to $log" >&3 exec >> "$log" 2>&1 final=0 - -echo "Trying to build pre-commit docker image" -if ! docker build --tag precommit . -then - final=1 - echo "pre-commit image failed to build" -else - if printf '%s\n' '#!/bin/sh' 'echo $1' | docker run -i precommit - - then - final=1 - echo "pre-commit image succeeds with incorrect example" - fi - - if ! printf '%s\n' '#!/bin/sh' 'echo "$1"' | docker run -i precommit - - then - final=1 - echo "pre-commit image fails with correct example" - fi -fi - -if [[ $final -ne 0 ]] -then - echo >&3 "pre-commit image failure, see log" -else - echo >&3 "pre-commit image succeeded" -fi - while read -r distro setup do [[ "$distro" = "#"* || -z "$distro" ]] && continue From b5da99c6b0978880c948fbdf2b3b3cf449bfdb59 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 29 Aug 2021 12:28:58 -0700 Subject: [PATCH 180/459] Add pre-commit instructions --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index a155b83..8de2194 100644 --- a/README.md +++ b/README.md @@ -239,6 +239,20 @@ pandoc -s -f markdown-smart -t man shellcheck.1.md -o shellcheck.1 sudo mv shellcheck.1 /usr/share/man/man1 ``` +### pre-commit + +To run ShellCheck via [pre-commit](https://pre-commit.com/), add the Docker image to your `.pre-commit-config.yaml`: + +``` +- repo: local + hooks: + - id: shellcheck + name: shellcheck + language: docker_image + entry: koalaman/shellcheck:stable # or e.g. ":v0.7.2" + types: [shell] +``` + ### Travis CI Travis CI has now integrated ShellCheck by default, so you don't need to manually install it. From 10817533d6fa0a39804ac69e540084aa3037ceab Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 29 Aug 2021 17:08:09 -0700 Subject: [PATCH 181/459] Add shellcheck-precommit hook to README.md --- README.md | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 8de2194..d91f962 100644 --- a/README.md +++ b/README.md @@ -241,16 +241,15 @@ sudo mv shellcheck.1 /usr/share/man/man1 ### pre-commit -To run ShellCheck via [pre-commit](https://pre-commit.com/), add the Docker image to your `.pre-commit-config.yaml`: +To run ShellCheck via [pre-commit](https://pre-commit.com/), add the hook to your `.pre-commit-config.yaml`: ``` -- repo: local - hooks: - - id: shellcheck - name: shellcheck - language: docker_image - entry: koalaman/shellcheck:stable # or e.g. ":v0.7.2" - types: [shell] +repos: +- repo: https://github.com/koalaman/shellcheck-precommit + rev: v0.7.2 + hooks: + - id: shellcheck +# args: ["--severity=warning"] # Optionally only show errors and warnings ``` ### Travis CI From f5fd9c2fed8bf99e114ba109d9ea452935f5726e Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 30 Aug 2021 10:56:55 -0700 Subject: [PATCH 182/459] Improve warnings about unnecessary subshells (fixes #2169) --- src/ShellCheck/Analytics.hs | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index f35fc0d..aff5048 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -3742,21 +3742,27 @@ checkForLoopGlobVariables _ t = suggest t = info (getId t) 2231 "Quote expansions in this for loop glob to prevent wordsplitting, e.g. \"$dir\"/*.txt ." + prop_checkSubshelledTests1 = verify checkSubshelledTests "a && ( [ b ] || ! [ c ] )" prop_checkSubshelledTests2 = verify checkSubshelledTests "( [ a ] )" prop_checkSubshelledTests3 = verify checkSubshelledTests "( [ a ] && [ b ] || test c )" prop_checkSubshelledTests4 = verify checkSubshelledTests "( [ a ] && { [ b ] && [ c ]; } )" +prop_checkSubshelledTests5 = verifyNot checkSubshelledTests "( [[ ${var:=x} = y ]] )" +prop_checkSubshelledTests6 = verifyNot checkSubshelledTests "( [[ $((i++)) = 10 ]] )" +prop_checkSubshelledTests7 = verifyNot checkSubshelledTests "( [[ $((i+=1)) = 10 ]] )" +prop_checkSubshelledTests8 = verify checkSubshelledTests "# shellcheck disable=SC2234\nf() ( [[ x ]] )" + checkSubshelledTests params t = case t of - T_Subshell id list | all isTestStructure list -> + T_Subshell id list | all isTestStructure list && (not (hasAssignment t)) -> case () of -- Special case for if (test) and while (test) _ | isCompoundCondition (getPath (parentMap params) t) -> - style id 2233 "Remove superfluous (..) around condition." + style id 2233 "Remove superfluous (..) around condition to avoid subshell overhead." - -- Special case for ([ x ]) - _ | isSingleTest list -> - style id 2234 "Remove superfluous (..) around test command." + -- Special case for ([ x ]), except for func() ( [ x ] ) + _ | isSingleTest list && (not $ isFunctionBody (getPath (parentMap params) t)) -> + style id 2234 "Remove superfluous (..) around test command to avoid subshell overhead." -- General case for ([ x ] || [ y ] && etc) _ -> style id 2235 "Use { ..; } instead of (..) to avoid subshell overhead." @@ -3768,6 +3774,11 @@ checkSubshelledTests params t = [c] | isTestCommand c -> True _ -> False + isFunctionBody path = + case path of + (_:f:_) -> isFunction f + _ -> False + isTestStructure t = case t of T_Banged _ t -> isTestStructure t @@ -3798,6 +3809,19 @@ checkSubshelledTests params t = T_UntilExpression {} : _ -> True _ -> False + hasAssignment t = isNothing $ doAnalysis guardNotAssignment t + guardNotAssignment t = + case t of + TA_Assignment {} -> Nothing + TA_Unary _ s _ -> guard . not $ "++" `isInfixOf` s || "--" `isInfixOf` s + T_DollarBraced _ _ l -> + let str = concat $ oversimplify l + modifier = getBracedModifier str + in + guard . not $ "=" `isPrefixOf` modifier || ":=" `isPrefixOf` modifier + T_DollarBraceCommandExpansion {} -> Nothing + _ -> Just () + -- Skip any parent of a T_Subshell until we reach something interesting skippable t = case t of From 747bd8fd6af31f578bf8683ac9df9f29d5ca6d0b Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 30 Aug 2021 19:50:00 -0700 Subject: [PATCH 183/459] Warn about strings for numerical operators in [[ ]] (fixes #2312) --- src/ShellCheck/Analytics.hs | 46 +++++++++++++++++++++++++++++++------ 1 file changed, 39 insertions(+), 7 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index aff5048..cc554e0 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1109,6 +1109,12 @@ prop_checkNumberComparisons13 = verify checkNumberComparisons "[ $foo > $bar ]" prop_checkNumberComparisons14 = verifyNot checkNumberComparisons "[[ foo < bar ]]" prop_checkNumberComparisons15 = verifyNot checkNumberComparisons "[ $foo '>' $bar ]" prop_checkNumberComparisons16 = verify checkNumberComparisons "[ foo -eq 'y' ]" +prop_checkNumberComparisons17 = verify checkNumberComparisons "[[ 'foo' -eq 2 ]]" +prop_checkNumberComparisons18 = verify checkNumberComparisons "[[ foo -eq 2 ]]" +prop_checkNumberComparisons19 = verifyNot checkNumberComparisons "foo=1; [[ foo -eq 2 ]]" +prop_checkNumberComparisons20 = verify checkNumberComparisons "[[ 2 -eq / ]]" +prop_checkNumberComparisons21 = verify checkNumberComparisons "[[ foo -eq foo ]]" + checkNumberComparisons params (TC_Binary id typ op lhs rhs) = do if isNum lhs || isNum rhs then do @@ -1134,8 +1140,8 @@ checkNumberComparisons params (TC_Binary id typ op lhs rhs) = do when (op `elem` arithmeticBinaryTestOps) $ do mapM_ checkDecimals [lhs, rhs] - when (typ == SingleBracket) $ - checkStrings [lhs, rhs] + mapM_ checkString [lhs, rhs] + where hasStringComparison = shellType params /= Sh @@ -1148,19 +1154,45 @@ checkNumberComparisons params (TC_Binary id typ op lhs rhs) = do decimalError = "Decimals are not supported. " ++ "Either use integers only, or use bc or awk to compare." - checkStrings = - mapM_ stringError . find isNonNum + checkString t = + let + asString = getLiteralStringDef "\0" t + isVar = isVariableName asString + kind = if isVar then "a variable" else "an arithmetic expression" + fix = if isVar then "$var" else "$((expr))" + in + when (isNonNum t) $ + if typ == SingleBracket + then + err (getId t) 2170 $ + "Invalid number for " ++ op ++ ". Use " ++ seqv op ++ + " to compare as string (or use " ++ fix ++ + " to expand as " ++ kind ++ ")." + else + -- We should warn if any of the following holds: + -- The string is not a variable name + -- Any part of it is quoted + -- It's not a recognized variable name + when (not isVar || any isQuotes (getWordParts t) || asString `notElem` assignedVariables) $ + warn (getId t) 2309 $ + op ++ " treats this as " ++ kind ++ ". " ++ + "Use " ++ seqv op ++ " to compare as string (or expand explicitly with " ++ fix ++ ")." + + assignedVariables :: [String] + assignedVariables = mapMaybe f (variableFlow params) + where + f t = do + Assignment (_, _, name, _) <- return t + return name isNonNum t = not . all numChar $ onlyLiteralString t numChar x = isDigit x || x `elem` "+-. " - stringError t = err (getId t) 2170 $ - "Numerical " ++ op ++ " does not dereference in [..]. Expand or use string operator." - isNum t = case oversimplify t of [v] -> all isDigit v _ -> False + isFraction t = case oversimplify t of [v] -> isJust $ matchRegex floatRegex v From 40216487d643c5c8a636d3e8a1a20535c1c478bf Mon Sep 17 00:00:00 2001 From: Fabian Wolff Date: Thu, 2 Sep 2021 17:47:06 +0200 Subject: [PATCH 184/459] Do not suggest `grep -c` as a replacement for `grep -l/-L | wc -l` --- src/ShellCheck/Analytics.hs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index cc554e0..3d854aa 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -508,6 +508,8 @@ prop_checkPipePitfalls13 = verifyNot checkPipePitfalls "foo | grep bar | wc -c" prop_checkPipePitfalls14 = verifyNot checkPipePitfalls "foo | grep -o bar | wc -cmwL" prop_checkPipePitfalls15 = verifyNot checkPipePitfalls "foo | grep bar | wc -cmwL" prop_checkPipePitfalls16 = verifyNot checkPipePitfalls "foo | grep -r bar | wc -l" +prop_checkPipePitfalls17 = verifyNot checkPipePitfalls "foo | grep -l bar | wc -l" +prop_checkPipePitfalls18 = verifyNot checkPipePitfalls "foo | grep -L bar | wc -l" checkPipePitfalls _ (T_Pipeline id _ commands) = do for ["find", "xargs"] $ \(find:xargs:_) -> @@ -529,7 +531,9 @@ checkPipePitfalls _ (T_Pipeline id _ commands) = do let flagsGrep = maybe [] (map snd . getAllFlags) $ getCommand grep flagsWc = maybe [] (map snd . getAllFlags) $ getCommand wc in - unless (any (`elem` ["o", "only-matching", "r", "R", "recursive"]) flagsGrep || any (`elem` ["m", "chars", "w", "words", "c", "bytes", "L", "max-line-length"]) flagsWc || null flagsWc) $ + unless (any (`elem` ["l", "files-with-matches", "L", "files-without-matches", "o", "only-matching", "r", "R", "recursive"]) flagsGrep + || any (`elem` ["m", "chars", "w", "words", "c", "bytes", "L", "max-line-length"]) flagsWc + || null flagsWc) $ style (getId grep) 2126 "Consider using grep -c instead of grep|wc -l." didLs <- fmap or . sequence $ [ From dc9032fca553d13340087d5094323540c1359d1b Mon Sep 17 00:00:00 2001 From: Christian Nassif-Haynes Date: Mon, 23 Aug 2021 03:27:40 +1000 Subject: [PATCH 185/459] Show info about `set -e` suppression during function calls --- src/ShellCheck/Analytics.hs | 98 ++++++++++++++++++++++++++++++----- src/ShellCheck/AnalyzerLib.hs | 22 ++++++-- 2 files changed, 103 insertions(+), 17 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index e1e55fd..b524410 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -253,6 +253,13 @@ optionalTreeChecks = [ cdPositive = "[ -e /etc/issue ]", cdNegative = "[[ -e /etc/issue ]]" }, checkRequireDoubleBracket) + + ,(newCheckDescription { + cdName = "check-set-e-suppressed", + cdDescription = "Notify when set -e is suppressed during function invocation", + cdPositive = "set -e; func() { cp *.txt ~/backup; rm *.txt; }; func && echo ok", + cdNegative = "set -e; func() { cp *.txt ~/backup; rm *.txt; }; func; echo ok" + }, checkSetESuppressed) ] optionalCheckMap :: Map.Map String (Parameters -> Token -> [TokenComment]) @@ -393,6 +400,24 @@ replaceToken id params r = surroundWith id params s = fixWith [replaceStart id params 0 s, replaceEnd id params 0 s] fixWith fixes = newFix { fixReplacements = fixes } +analyse f t = execState (doAnalysis f t) [] + +-- Make a map from functions to definition IDs +functions t = Map.fromList $ analyse findFunctions t +findFunctions (T_Function id _ _ name _) + = modify ((name, id):) +findFunctions _ = return () + +-- Make a map from aliases to definition IDs +aliases t = Map.fromList $ analyse findAliases t +findAliases t@(T_SimpleCommand _ _ (_:args)) + | t `isUnqualifiedCommand` "alias" = mapM_ getAlias args +findAliases _ = return () +getAlias arg = + let string = onlyLiteralString arg + in when ('=' `elem` string) $ + modify ((takeWhile (/= '=') string, getId arg):) + prop_checkEchoWc3 = verify checkEchoWc "n=$(echo $foo | wc -c)" checkEchoWc _ (T_Pipeline id _ [a, b]) = when (acmd == ["echo", "${VAR}"]) $ @@ -2239,21 +2264,11 @@ checkFunctionsUsedExternally params t = findExecFlags = ["-exec", "-execdir", "-ok"] dropFlags = dropWhile (\x -> "-" `isPrefixOf` fst x) - -- Make a map from functions/aliases to definition IDs - analyse f t = execState (doAnalysis f t) [] - functions = Map.fromList $ analyse findFunctions t - findFunctions (T_Function id _ _ name _) = modify ((name, id):) - findFunctions t@(T_SimpleCommand id _ (_:args)) - | t `isUnqualifiedCommand` "alias" = mapM_ getAlias args - findFunctions _ = return () - getAlias arg = - let string = onlyLiteralString arg - in when ('=' `elem` string) $ - modify ((takeWhile (/= '=') string, getId arg):) + functionsAndAliases = Map.union (functions t) (aliases t) checkArg cmd (_, arg) = sequence_ $ do literalArg <- getUnquotedLiteral arg -- only consider unquoted literals - definitionId <- Map.lookup literalArg functions + definitionId <- Map.lookup literalArg functionsAndAliases return $ do warn (getId arg) 2033 "Shell functions can't be passed to external commands." @@ -4583,6 +4598,65 @@ checkArrayValueUsedAsIndex params _ = _ -> Nothing +prop_checkSetESuppressed1 = verifyTree checkSetESuppressed "set -e; f(){ :; }; x=$(f)" +prop_checkSetESuppressed2 = verifyNotTree checkSetESuppressed "f(){ :; }; x=$(f)" +prop_checkSetESuppressed3 = verifyNotTree checkSetESuppressed "set -e; f(){ :; }; x=$(set -e; f)" +prop_checkSetESuppressed4 = verifyTree checkSetESuppressed "set -e; f(){ :; }; baz=$(set -e; f) || :" +prop_checkSetESuppressed5 = verifyNotTree checkSetESuppressed "set -e; f(){ :; }; baz=$(echo \"\") || :" +prop_checkSetESuppressed6 = verifyTree checkSetESuppressed "set -e; f(){ :; }; f && echo" +prop_checkSetESuppressed7 = verifyTree checkSetESuppressed "set -e; f(){ :; }; f || echo" +prop_checkSetESuppressed8 = verifyNotTree checkSetESuppressed "set -e; f(){ :; }; echo && f" +prop_checkSetESuppressed9 = verifyNotTree checkSetESuppressed "set -e; f(){ :; }; echo || f" +prop_checkSetESuppressed10 = verifyTree checkSetESuppressed "set -e; f(){ :; }; ! f" +prop_checkSetESuppressed11 = verifyTree checkSetESuppressed "set -e; f(){ :; }; if f; then :; fi" +prop_checkSetESuppressed12 = verifyTree checkSetESuppressed "set -e; f(){ :; }; if set -e; f; then :; fi" +prop_checkSetESuppressed13 = verifyTree checkSetESuppressed "set -e; f(){ :; }; while f; do :; done" +prop_checkSetESuppressed14 = verifyTree checkSetESuppressed "set -e; f(){ :; }; while set -e; f; do :; done" +prop_checkSetESuppressed15 = verifyTree checkSetESuppressed "set -e; f(){ :; }; until f; do :; done" +prop_checkSetESuppressed16 = verifyTree checkSetESuppressed "set -e; f(){ :; }; until set -e; f; do :; done" +prop_checkSetESuppressed17 = verifyNotTree checkSetESuppressed "set -e; f(){ :; }; g(){ :; }; g f" +prop_checkSetESuppressed18 = verifyNotTree checkSetESuppressed "set -e; shopt -s inherit_errexit; f(){ :; }; x=$(f)" +checkSetESuppressed params t = + if hasSetE params then runNodeAnalysis checkNode params t else [] + where + checkNode _ (T_SimpleCommand _ _ (cmd:_)) = when (isFunction cmd) (checkCmd cmd) + checkNode _ _ = return () + + functions_ = functions t + + isFunction cmd = isJust $ do + literalArg <- getUnquotedLiteral cmd + Map.lookup literalArg functions_ + + checkCmd cmd = go $ getPath (parentMap params) cmd + where + go (child:parent:rest) = do + case parent of + T_Banged _ condition | child `isIn` [condition] -> informConditional "a ! condition" cmd + T_AndIf _ condition _ | child `isIn` [condition] -> informConditional "an && condition" cmd + T_OrIf _ condition _ | child `isIn` [condition] -> informConditional "an || condition" cmd + T_IfExpression _ condition _ | child `isIn` concatMap fst condition -> informConditional "an 'if' condition" cmd + T_UntilExpression _ condition _ | child `isIn` condition -> informConditional "an 'until' condition" cmd + T_WhileExpression _ condition _ | child `isIn` condition -> informConditional "a 'while' condition" cmd + T_DollarExpansion {} | not $ errExitEnabled parent -> informUninherited cmd + T_Backticked {} | not $ errExitEnabled parent -> informUninherited cmd + _ -> return () + go (parent:rest) + go _ = return () + + informConditional condType t = + info (getId t) 2310 ( + "This function is invoked in " ++ condType ++ " so set -e " ++ + "will be disabled. Invoke separately if failures should " ++ + "cause the script to exit.") + informUninherited t = + info (getId t) 2311 ( + "Bash implicitly disabled set -e for this function " ++ + "invocation because it's inside a command substitution. " ++ + "Add set -e; before it or enable inherit_errexit.") + errExitEnabled t = hasInheritErrexit params || containsSetE t + isIn t cmds = getId t `elem` map getId cmds + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 633543a..42d6f73 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -79,6 +79,8 @@ composeAnalyzers f g x = f x >> g x data Parameters = Parameters { -- Whether this script has the 'lastpipe' option set/default. hasLastpipe :: Bool, + -- Whether this script has the 'inherit_errexit' option set/default. + hasInheritErrexit :: Bool, -- Whether this script has 'set -e' anywhere. hasSetE :: Bool, -- A linear (bad) analysis of data flow @@ -196,7 +198,12 @@ makeParameters spec = Dash -> False Sh -> False Ksh -> True, - + hasInheritErrexit = + case shellType params of + Bash -> containsInheritErrexit root + Dash -> True + Sh -> True + Ksh -> False, shellTypeSpecified = isJust (asShellType spec) || isJust (asFallbackShell spec), parentMap = getParentTree root, variableFlow = getVariableFlow params root, @@ -219,18 +226,23 @@ containsSetE root = isNothing $ doAnalysis (guard . not . isSetE) root _ -> False re = mkRegex "[[:space:]]-[^-]*e" --- Does this script mention 'shopt -s lastpipe' anywhere? --- Also used as a hack. -containsLastpipe root = +containsShopt shopt root = isNothing $ doAnalysis (guard . not . isShoptLastPipe) root where isShoptLastPipe t = case t of T_SimpleCommand {} -> t `isUnqualifiedCommand` "shopt" && - ("lastpipe" `elem` oversimplify t) + (shopt `elem` oversimplify t) _ -> False +-- Does this script mention 'shopt -s inherit_errexit' anywhere? +containsInheritErrexit = containsShopt "inherit_errexit" + +-- Does this script mention 'shopt -s lastpipe' anywhere? +-- Also used as a hack. +containsLastpipe = containsShopt "lastpipe" + prop_determineShell0 = determineShellTest "#!/bin/sh" == Sh prop_determineShell1 = determineShellTest "#!/usr/bin/env ksh" == Ksh From 4e703e5c61c6366bfd486d728bc624025e344e68 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 15 Sep 2021 18:02:37 -0700 Subject: [PATCH 186/459] Allow specifying external-sources=true in shellcheckrc (fixes #1818) --- CHANGELOG.md | 4 +- shellcheck.1.md | 14 +++++ shellcheck.hs | 24 +++++---- src/ShellCheck/AST.hs | 1 + src/ShellCheck/Checker.hs | 75 ++++++++++++++++++++++++-- src/ShellCheck/Formatter/CheckStyle.hs | 2 +- src/ShellCheck/Formatter/Diff.hs | 2 +- src/ShellCheck/Formatter/GCC.hs | 2 +- src/ShellCheck/Formatter/JSON1.hs | 2 +- src/ShellCheck/Formatter/TTY.hs | 2 +- src/ShellCheck/Interface.hs | 14 +++-- src/ShellCheck/Parser.hs | 34 +++++++++--- 12 files changed, 145 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ae3cae..4dc25f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ -## Git +## Git (0.8.0) ### Added +- `external-sources=true` directive can be added to .shellcheckrc to make + shellcheck behave as if `-x` was specified. - SC2286-SC2288: Warn when command name ends in a symbol like `/.)'"` - SC2289: Warn when command name contains tabs or linefeeds - SC2291: Warn about repeated unquoted spaces between words in echo diff --git a/shellcheck.1.md b/shellcheck.1.md index d038df2..070c3a4 100644 --- a/shellcheck.1.md +++ b/shellcheck.1.md @@ -112,6 +112,9 @@ not warn at all, as `ksh` supports decimals in arithmetic contexts. line (plus `/dev/null`). This option allows following any file the script may `source`. + This option may also be enabled using `external-sources=true` in + `.shellcheckrc`. This flag takes precedence. + **FILES...** : One or more script files to check, or "-" for standard input. @@ -239,6 +242,14 @@ Valid keys are: : Enable an optional check by name, as listed with **--list-optional**. Only file-wide `enable` directives are considered. +**external-sources** +: Set to `true` in `.shellcheckrc` to always allow ShellCheck to open + arbitrary files from 'source' statements (the way most tools do). + + This option defaults to `false` only due to ShellCheck's origin as a + remote service for checking untrusted scripts. It can safely be enabled + for normal development. + **source** : Overrides the filename included by a `source`/`.` statement. This can be used to tell shellcheck where to look for a file whose name is determined @@ -270,6 +281,9 @@ Here is an example `.shellcheckrc`: source-path=SCRIPTDIR source-path=/mnt/chroot + # Allow opening any 'source'd file, even if not specified as input + external-sources=true + # Turn on warnings for unquoted variables with safe values enable=quote-safe-variables diff --git a/shellcheck.hs b/shellcheck.hs index d7e818d..bf70445 100644 --- a/shellcheck.hs +++ b/shellcheck.hs @@ -234,7 +234,7 @@ runFormatter sys format options files = do process :: FilePath -> IO Status process filename = do - input <- siReadFile sys filename + input <- siReadFile sys Nothing filename either (reportFailure filename) check input where check contents = do @@ -389,6 +389,7 @@ parseOption flag options = throwError SyntaxFailure return (Prelude.read num :: Integer) +ioInterface :: Options -> [FilePath] -> IO (SystemInterface IO) ioInterface options files = do inputs <- mapM normalize files cache <- newIORef emptyCache @@ -402,14 +403,14 @@ ioInterface options files = do emptyCache :: Map.Map FilePath String emptyCache = Map.empty - get cache inputs file = do + get cache inputs rcSuggestsExternal file = do map <- readIORef cache case Map.lookup file map of Just x -> return $ Right x - Nothing -> fetch cache inputs file + Nothing -> fetch cache inputs rcSuggestsExternal file - fetch cache inputs file = do - ok <- allowable inputs file + fetch cache inputs rcSuggestsExternal file = do + ok <- allowable rcSuggestsExternal inputs file if ok then (do (contents, shouldCache) <- inputFile file @@ -417,13 +418,16 @@ ioInterface options files = do modifyIORef cache $ Map.insert file contents return $ Right contents ) `catch` handler - else return $ Left (file ++ " was not specified as input (see shellcheck -x).") + else + if rcSuggestsExternal == Just False + then return $ Left (file ++ " was not specified as input, and external files were disabled via directive.") + else return $ Left (file ++ " was not specified as input (see shellcheck -x).") where handler :: IOException -> IO (Either ErrorMessage String) handler ex = return . Left $ show ex - allowable inputs x = - if externalSources options + allowable rcSuggestsExternal inputs x = + if fromMaybe (externalSources options) rcSuggestsExternal then return True else do path <- normalize x @@ -497,7 +501,7 @@ ioInterface options files = do b <- p x if b then pure (Just x) else acc - findSourceFile inputs sourcePathFlag currentScript sourcePathAnnotation original = + findSourceFile inputs sourcePathFlag currentScript rcSuggestsExternal sourcePathAnnotation original = if isAbsolute original then let (_, relative) = splitDrive original @@ -506,7 +510,7 @@ ioInterface options files = do find original original where find filename deflt = do - sources <- findM ((allowable inputs) `andM` doesFileExist) $ + sources <- findM ((allowable rcSuggestsExternal inputs) `andM` doesFileExist) $ (adjustPath filename):(map (( filename) . adjustPath) $ sourcePathFlag ++ sourcePathAnnotation) case sources of Nothing -> return deflt diff --git a/src/ShellCheck/AST.hs b/src/ShellCheck/AST.hs index 52b6e17..2cd2f6f 100644 --- a/src/ShellCheck/AST.hs +++ b/src/ShellCheck/AST.hs @@ -150,6 +150,7 @@ data Annotation = | SourceOverride String | ShellOverride String | SourcePath String + | ExternalSources Bool deriving (Show, Eq) data ConditionType = DoubleBracket | SingleBracket deriving (Show, Eq) diff --git a/src/ShellCheck/Checker.hs b/src/ShellCheck/Checker.hs index d81d664..514f97d 100644 --- a/src/ShellCheck/Checker.hs +++ b/src/ShellCheck/Checker.hs @@ -156,6 +156,11 @@ checkWithIncludesAndSourcePath includes mapper = getErrors siFindSource = mapper } +checkWithRcIncludesAndSourcePath rc includes mapper = getErrors + (mockRcFile rc $ mockedSystemInterface includes) { + siFindSource = mapper + } + prop_findsParseIssue = check "echo \"$12\"" == [1037] prop_commentDisablesParseIssue1 = @@ -384,7 +389,7 @@ prop_canEnableOptionalsWithRc = result == [2244] prop_sourcePathRedirectsName = result == [2086] where - f "dir/myscript" _ "lib" = return "foo/lib" + f "dir/myscript" _ _ "lib" = return "foo/lib" result = checkWithIncludesAndSourcePath [("foo/lib", "echo $1")] f emptyCheckSpec { csScript = "#!/bin/bash\nsource lib", csFilename = "dir/myscript", @@ -393,7 +398,7 @@ prop_sourcePathRedirectsName = result == [2086] prop_sourcePathAddsAnnotation = result == [2086] where - f "dir/myscript" ["mypath"] "lib" = return "foo/lib" + f "dir/myscript" _ ["mypath"] "lib" = return "foo/lib" result = checkWithIncludesAndSourcePath [("foo/lib", "echo $1")] f emptyCheckSpec { csScript = "#!/bin/bash\n# shellcheck source-path=mypath\nsource lib", csFilename = "dir/myscript", @@ -402,13 +407,75 @@ prop_sourcePathAddsAnnotation = result == [2086] prop_sourcePathRedirectsDirective = result == [2086] where - f "dir/myscript" _ "lib" = return "foo/lib" - f _ _ _ = return "/dev/null" + f "dir/myscript" _ _ "lib" = return "foo/lib" + f _ _ _ _ = return "/dev/null" result = checkWithIncludesAndSourcePath [("foo/lib", "echo $1")] f emptyCheckSpec { csScript = "#!/bin/bash\n# shellcheck source=lib\nsource kittens", csFilename = "dir/myscript", csCheckSourced = True } +prop_rcCanAllowExternalSources = result == [2086] + where + f "dir/myscript" (Just True) _ "mylib" = return "resolved/mylib" + f a b c d = error $ show ("Unexpected", a, b, c, d) + result = checkWithRcIncludesAndSourcePath "external-sources=true" [("resolved/mylib", "echo $1")] f emptyCheckSpec { + csScript = "#!/bin/bash\nsource mylib", + csFilename = "dir/myscript", + csCheckSourced = True + } + +prop_rcCanDenyExternalSources = result == [2086] + where + f "dir/myscript" (Just False) _ "mylib" = return "resolved/mylib" + f a b c d = error $ show ("Unexpected", a, b, c, d) + result = checkWithRcIncludesAndSourcePath "external-sources=false" [("resolved/mylib", "echo $1")] f emptyCheckSpec { + csScript = "#!/bin/bash\nsource mylib", + csFilename = "dir/myscript", + csCheckSourced = True + } + +prop_rcCanLeaveExternalSourcesUnspecified = result == [2086] + where + f "dir/myscript" Nothing _ "mylib" = return "resolved/mylib" + f a b c d = error $ show ("Unexpected", a, b, c, d) + result = checkWithRcIncludesAndSourcePath "" [("resolved/mylib", "echo $1")] f emptyCheckSpec { + csScript = "#!/bin/bash\nsource mylib", + csFilename = "dir/myscript", + csCheckSourced = True + } + +prop_fileCanDisableExternalSources = result == [2006, 2086] + where + f "dir/myscript" (Just True) _ "withExternal" = return "withExternal" + f "dir/myscript" (Just False) _ "withoutExternal" = return "withoutExternal" + f a b c d = error $ show ("Unexpected", a, b, c, d) + result = checkWithRcIncludesAndSourcePath "external-sources=true" [("withExternal", "echo $1"), ("withoutExternal", "_=`foo`")] f emptyCheckSpec { + csScript = "#!/bin/bash\ntrue\nsource withExternal\n# shellcheck external-sources=false\nsource withoutExternal", + csFilename = "dir/myscript", + csCheckSourced = True + } + +prop_fileCannotEnableExternalSources = result == [1144] + where + f "dir/myscript" Nothing _ "foo" = return "foo" + f a b c d = error $ show ("Unexpected", a, b, c, d) + result = checkWithRcIncludesAndSourcePath "" [("foo", "true")] f emptyCheckSpec { + csScript = "#!/bin/bash\n# shellcheck external-sources=true\nsource foo", + csFilename = "dir/myscript", + csCheckSourced = True + } + +prop_fileCannotEnableExternalSources2 = result == [1144] + where + f "dir/myscript" (Just False) _ "foo" = return "foo" + f a b c d = error $ show ("Unexpected", a, b, c, d) + result = checkWithRcIncludesAndSourcePath "external-sources=false" [("foo", "true")] f emptyCheckSpec { + csScript = "#!/bin/bash\n# shellcheck external-sources=true\nsource foo", + csFilename = "dir/myscript", + csCheckSourced = True + } + + return [] runTests = $quickCheckAll diff --git a/src/ShellCheck/Formatter/CheckStyle.hs b/src/ShellCheck/Formatter/CheckStyle.hs index f3fea88..c79ac21 100644 --- a/src/ShellCheck/Formatter/CheckStyle.hs +++ b/src/ShellCheck/Formatter/CheckStyle.hs @@ -48,7 +48,7 @@ outputResults cr sys = fileGroups = groupWith sourceFile comments outputGroup group = do let filename = sourceFile (head group) - result <- (siReadFile sys) filename + result <- siReadFile sys (Just True) filename let contents = either (const "") id result outputFile filename contents group diff --git a/src/ShellCheck/Formatter/Diff.hs b/src/ShellCheck/Formatter/Diff.hs index 83fb232..9e31780 100644 --- a/src/ShellCheck/Formatter/Diff.hs +++ b/src/ShellCheck/Formatter/Diff.hs @@ -90,7 +90,7 @@ reportResult foundIssues reportedIssues color result sys = do mapM_ output $ M.toList fixmap where output (name, fix) = do - file <- (siReadFile sys) name + file <- siReadFile sys (Just True) name case file of Right contents -> do putStrLn $ formatDoc color $ makeDiff name contents fix diff --git a/src/ShellCheck/Formatter/GCC.hs b/src/ShellCheck/Formatter/GCC.hs index 9c5fa5f..5106e4c 100644 --- a/src/ShellCheck/Formatter/GCC.hs +++ b/src/ShellCheck/Formatter/GCC.hs @@ -43,7 +43,7 @@ outputAll cr sys = mapM_ f groups f :: [PositionedComment] -> IO () f group = do let filename = sourceFile (head group) - result <- (siReadFile sys) filename + result <- siReadFile sys (Just True) filename let contents = either (const "") id result outputResult filename contents group diff --git a/src/ShellCheck/Formatter/JSON1.hs b/src/ShellCheck/Formatter/JSON1.hs index 7335d8c..54aad34 100644 --- a/src/ShellCheck/Formatter/JSON1.hs +++ b/src/ShellCheck/Formatter/JSON1.hs @@ -117,7 +117,7 @@ collectResult ref cr sys = mapM_ f groups f :: [PositionedComment] -> IO () f group = do let filename = sourceFile (head group) - result <- siReadFile sys filename + result <- siReadFile sys (Just True) filename let contents = either (const "") id result let comments' = makeNonVirtual comments contents modifyIORef ref (\x -> comments' ++ x) diff --git a/src/ShellCheck/Formatter/TTY.hs b/src/ShellCheck/Formatter/TTY.hs index 0d474d7..bb57894 100644 --- a/src/ShellCheck/Formatter/TTY.hs +++ b/src/ShellCheck/Formatter/TTY.hs @@ -121,7 +121,7 @@ outputResult options ref result sys = do outputForFile color sys comments = do let fileName = sourceFile (head comments) - result <- (siReadFile sys) fileName + result <- siReadFile sys (Just True) fileName let contents = either (const "") id result let fileLinesList = lines contents let lineCount = length fileLinesList diff --git a/src/ShellCheck/Interface.hs b/src/ShellCheck/Interface.hs index 87346a1..7528559 100644 --- a/src/ShellCheck/Interface.hs +++ b/src/ShellCheck/Interface.hs @@ -73,14 +73,18 @@ import qualified Data.Map as Map data SystemInterface m = SystemInterface { - -- | Read a file by filename, or return an error - siReadFile :: String -> m (Either ErrorMessage String), + -- | Given: + -- What annotations say about including external files (if anything) + -- A resolved filename from siFindSource + -- Read the file or return an error + siReadFile :: Maybe Bool -> String -> m (Either ErrorMessage String), -- | Given: -- the current script, + -- what annotations say about including external files (if anything) -- a list of source-path annotations in effect, -- and a sourced file, -- find the sourced file - siFindSource :: String -> [String] -> String -> m FilePath, + siFindSource :: String -> Maybe Bool -> [String] -> String -> m FilePath, -- | Get the configuration file (name, contents) for a filename siGetConfig :: String -> m (Maybe (FilePath, String)) } @@ -313,11 +317,11 @@ mockedSystemInterface files = SystemInterface { siGetConfig = const $ return Nothing } where - rf file = return $ + rf _ file = return $ case find ((== file) . fst) files of Nothing -> Left "File not included in mock." Just (_, contents) -> Right contents - fs _ _ file = return file + fs _ _ _ file = return file mockRcFile rcfile mock = mock { siGetConfig = const . return $ Just (".shellcheckrc", rcfile) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 45434a8..b59ebc2 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -987,9 +987,9 @@ prop_readAnnotation7 = isOk readAnnotation "# shellcheck disable=SC1000,SC2000-S readAnnotation = called "shellcheck directive" $ do try readAnnotationPrefix many1 linewhitespace - readAnnotationWithoutPrefix + readAnnotationWithoutPrefix True -readAnnotationWithoutPrefix = do +readAnnotationWithoutPrefix sandboxed = do values <- many1 readKey optional readAnyComment void linefeed <|> eof <|> do @@ -1035,6 +1035,21 @@ readAnnotationWithoutPrefix = do "This shell type is unknown. Use e.g. sh or bash." return [ShellOverride shell] + "external-sources" -> do + pos <- getPosition + value <- many1 letter + case value of + "true" -> + if sandboxed + then do + parseNoteAt pos ErrorC 1144 "external-sources can only be enabled in .shellcheckrc, not in individual files." + return [] + else return [ExternalSources True] + "false" -> return [ExternalSources False] + _ -> do + parseNoteAt pos ErrorC 1145 "Unknown external-sources value. Expected true/false." + return [] + _ -> do parseNoteAt keyPos WarningC 1107 "This directive is unknown. It will be ignored." anyChar `reluctantlyTill` whitespace @@ -2176,10 +2191,12 @@ readSource t@(T_Redirecting _ _ (T_SimpleCommand cmdId _ (cmd:file':rest'))) = d if filename == "/dev/null" -- always allow /dev/null then return (Right "", filename) else do + allAnnotations <- getCurrentAnnotations True currentScript <- Mr.asks currentFilename - paths <- mapMaybe getSourcePath <$> getCurrentAnnotations True - resolved <- system $ siFindSource sys currentScript paths filename - contents <- system $ siReadFile sys resolved + let paths = mapMaybe getSourcePath allAnnotations + let externalSources = listToMaybe $ mapMaybe getExternalSources allAnnotations + resolved <- system $ siFindSource sys currentScript externalSources paths filename + contents <- system $ siReadFile sys externalSources resolved return (contents, resolved) case input of Left err -> do @@ -2213,6 +2230,11 @@ readSource t@(T_Redirecting _ _ (T_SimpleCommand cmdId _ (cmd:file':rest'))) = d SourcePath x -> Just x _ -> Nothing + getExternalSources t = + case t of + ExternalSources b -> Just b + _ -> Nothing + -- If the word has a single expansion as the directory, try stripping it -- This affects `$foo/bar` but not `${foo}-dir/bar` or `/foo/$file` stripDynamicPrefix word = @@ -3202,7 +3224,7 @@ prop_readConfigKVs4 = isOk readConfigKVs "\n\n\n\n\t \n" prop_readConfigKVs5 = isOk readConfigKVs "# shellcheck accepts annotation-like comments in rc files\ndisable=1234" readConfigKVs = do anySpacingOrComment - annotations <- many (readAnnotationWithoutPrefix <* anySpacingOrComment) + annotations <- many (readAnnotationWithoutPrefix False <* anySpacingOrComment) eof return $ concat annotations anySpacingOrComment = From 09aa15c9b7bc33135178c7fb3338a870f7ce6a88 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 18 Sep 2021 12:50:01 -0700 Subject: [PATCH 187/459] Allow `disable=all` to disable all warnings (fixes #2323) --- CHANGELOG.md | 1 + shellcheck.1.md | 1 + src/ShellCheck/Checker.hs | 7 +++++++ src/ShellCheck/Parser.hs | 7 ++++++- 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4dc25f3..125d0c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ## Git (0.8.0) ### Added +- `disable=all` now conveniently disables all warnings - `external-sources=true` directive can be added to .shellcheckrc to make shellcheck behave as if `-x` was specified. - SC2286-SC2288: Warn when command name ends in a symbol like `/.)'"` diff --git a/shellcheck.1.md b/shellcheck.1.md index 070c3a4..1103acc 100644 --- a/shellcheck.1.md +++ b/shellcheck.1.md @@ -237,6 +237,7 @@ Valid keys are: The command can be a simple command like `echo foo`, or a compound command like a function definition, subshell block or loop. A range can be be specified with a dash, e.g. `disable=SC3000-SC4000` to exclude 3xxx. + All warnings can be disabled with `disable=all`. **enable** : Enable an optional check by name, as listed with **--list-optional**. diff --git a/src/ShellCheck/Checker.hs b/src/ShellCheck/Checker.hs index 514f97d..cce1063 100644 --- a/src/ShellCheck/Checker.hs +++ b/src/ShellCheck/Checker.hs @@ -306,6 +306,13 @@ prop_canDisableShebangWarning = null $ result csScript = "#shellcheck disable=SC2148\nfoo" } +prop_canDisableAllWarnings = result == [2086] + where + result = checkWithSpec [] emptyCheckSpec { + csFilename = "file.sh", + csScript = "#!/bin/sh\necho $1\n#shellcheck disable=all\necho `echo $1`" + } + prop_canDisableParseErrors = null $ result where result = checkWithSpec [] emptyCheckSpec { diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index b59ebc2..4f26a80 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -984,6 +984,7 @@ prop_readAnnotation4 = isWarning readAnnotation "# shellcheck cats=dogs disable= prop_readAnnotation5 = isOk readAnnotation "# shellcheck disable=SC2002 # All cats are precious\n" prop_readAnnotation6 = isOk readAnnotation "# shellcheck disable=SC1234 # shellcheck foo=bar\n" prop_readAnnotation7 = isOk readAnnotation "# shellcheck disable=SC1000,SC2000-SC3000,SC1001\n" +prop_readAnnotation8 = isOk readAnnotation "# shellcheck disable=all\n" readAnnotation = called "shellcheck directive" $ do try readAnnotationPrefix many1 linewhitespace @@ -1004,8 +1005,12 @@ readAnnotationWithoutPrefix sandboxed = do key <- many1 (letter <|> char '-') char '=' <|> fail "Expected '=' after directive key" annotations <- case key of - "disable" -> readRange `sepBy` char ',' + "disable" -> readElement `sepBy` char ',' where + readElement = readRange <|> readAll + readAll = do + string "all" + return $ DisableComment 0 1000000 readRange = do from <- readCode to <- choice [ char '-' *> readCode, return $ from+1 ] From 253650706086662c8d655e80b2e3d0e50a3df837 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 18 Sep 2021 17:43:55 -0700 Subject: [PATCH 188/459] Remove SC1004 (fixes #2326) --- CHANGELOG.md | 3 +++ src/ShellCheck/Parser.hs | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 125d0c5..fef7680 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,9 @@ - Quote warnings are now emitted for declaration utilities in sh - Leading `_` can now be used to suppress warnings about unused variables +### Removed +- SC1003: Literal backslash+linefeed in '' was found to be usually correct + ## v0.7.2 - 2021-04-19 ### Added diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 4f26a80..30bbb69 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -1511,7 +1511,6 @@ readSingleEscaped = do case x of '\'' -> parseProblemAt pos InfoC 1003 "Want to escape a single quote? echo 'This is how it'\\''s done'."; - '\n' -> parseProblemAt pos InfoC 1004 "This backslash+linefeed is literal. Break outside single quotes if you just want to break the line." _ -> return () return [s] From 8012f6761d1d32b5a6d26f86b50ae639457f1b3d Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 18 Sep 2021 17:59:30 -0700 Subject: [PATCH 189/459] Suppress SC2094 when both are input redirections (fixes #2325) --- src/ShellCheck/Analytics.hs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 34bca94..b21a821 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -791,6 +791,7 @@ prop_checkRedirectToSame5 = verifyNot checkRedirectToSame "foo > bar 2> bar" prop_checkRedirectToSame6 = verifyNot checkRedirectToSame "echo foo > foo" prop_checkRedirectToSame7 = verifyNot checkRedirectToSame "sed 's/foo/bar/g' file | sponge file" prop_checkRedirectToSame8 = verifyNot checkRedirectToSame "while read -r line; do _=\"$fname\"; done <\"$fname\"" +prop_checkRedirectToSame9 = verifyNot checkRedirectToSame "while read -r line; do cat < \"$fname\"; done <\"$fname\"" checkRedirectToSame params s@(T_Pipeline _ _ list) = mapM_ (\l -> (mapM_ (\x -> doAnalysis (checkOccurrences x) l) (getAllRedirs list))) list where @@ -799,6 +800,7 @@ checkRedirectToSame params s@(T_Pipeline _ _ list) = checkOccurrences t@(T_NormalWord exceptId x) u@(T_NormalWord newId y) | exceptId /= newId && x == y + && not (isInput t && isInput u) && not (isOutput t && isOutput u) && not (special t) && not (any isHarmlessCommand [t,u]) @@ -817,6 +819,13 @@ checkRedirectToSame params s@(T_Pipeline _ _ list) = _ -> [] getRedirs _ = [] special x = "/dev/" `isPrefixOf` concat (oversimplify x) + isInput t = + case drop 1 $ getPath (parentMap params) t of + T_IoFile _ op _:_ -> + case op of + T_Less _ -> True + _ -> False + _ -> False isOutput t = case drop 1 $ getPath (parentMap params) t of T_IoFile _ op _:_ -> From b044f5b23ab79b40f6885f28eafd82666b9ee7d4 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 18 Sep 2021 18:49:58 -0700 Subject: [PATCH 190/459] Don't trigger SC2140 on ${x+"a" "b"} (fixes #2265) --- src/ShellCheck/Analytics.hs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index b21a821..49a6557 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1763,6 +1763,7 @@ prop_checkInexplicablyUnquoted6 = verifyNot checkInexplicablyUnquoted "\"$dir\"s prop_checkInexplicablyUnquoted7 = verifyNot checkInexplicablyUnquoted "${dir/\"foo\"/\"bar\"}" prop_checkInexplicablyUnquoted8 = verifyNot checkInexplicablyUnquoted " 'foo'\\\n 'bar'" prop_checkInexplicablyUnquoted9 = verifyNot checkInexplicablyUnquoted "[[ $x =~ \"foo\"(\"bar\"|\"baz\") ]]" +prop_checkInexplicablyUnquoted10 = verifyNot checkInexplicablyUnquoted "cmd ${x+--name=\"$x\" --output=\"$x.out\"}" checkInexplicablyUnquoted params (T_NormalWord id tokens) = mapM_ check (tails tokens) where check (T_SingleQuoted _ _:T_Literal id str:_) @@ -1774,19 +1775,20 @@ checkInexplicablyUnquoted params (T_NormalWord id tokens) = mapM_ check (tails t T_DollarExpansion id _ -> warnAboutExpansion id T_DollarBraced id _ _ -> warnAboutExpansion id T_Literal id s - | not (quotesSingleThing a && quotesSingleThing b || isRegex (getPath (parentMap params) trapped)) -> + | not (quotesSingleThing a && quotesSingleThing b || isSpecial (getPath (parentMap params) trapped)) -> warnAboutLiteral id _ -> return () check _ = return () -- Regexes for [[ .. =~ re ]] are parsed with metacharacters like ()| as unquoted - -- literals, so avoid overtriggering on these. - isRegex t = + -- literals. The same is true for ${x+"foo" "bar"}. Avoid overtriggering on these. + isSpecial t = case t of (T_Redirecting {} : _) -> False + T_DollarBraced {} : _ -> True (a:(TC_Binary _ _ "=~" lhs rhs):rest) -> getId a == getId rhs - _:rest -> isRegex rest + _:rest -> isSpecial rest _ -> False -- If the surrounding quotes quote single things, like "$foo"_and_then_some_"$stuff", From e7df718724341e1ff9ea010ab867afc0376f67b0 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 18 Sep 2021 19:22:46 -0700 Subject: [PATCH 191/459] Strip lines containing "STRIP" from ./striptests --- src/ShellCheck/Analytics.hs | 6 +++--- src/ShellCheck/Formatter/Diff.hs | 3 --- src/ShellCheck/Parser.hs | 10 +++++----- striptests | 1 + 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 49a6557..de59be1 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -42,7 +42,7 @@ import Data.List import Data.Maybe import Data.Ord import Data.Semigroup -import Debug.Trace +import Debug.Trace -- STRIP import qualified Data.Map.Strict as Map import Test.QuickCheck.All (forAllProperties) import Test.QuickCheck.Test (quickCheckWithResult, stdArgs, maxSuccess) @@ -1011,8 +1011,8 @@ checkStderrRedirect params redir@(T_Redirecting _ [ checkStderrRedirect _ _ = return () -lt x = trace ("Tracing " ++ show x) x -ltt t = trace ("Tracing " ++ show t) +lt x = trace ("Tracing " ++ show x) x -- STRIP +ltt t = trace ("Tracing " ++ show t) -- STRIP prop_checkSingleQuotedVariables = verify checkSingleQuotedVariables "echo '$foo'" diff --git a/src/ShellCheck/Formatter/Diff.hs b/src/ShellCheck/Formatter/Diff.hs index 9e31780..197b3af 100644 --- a/src/ShellCheck/Formatter/Diff.hs +++ b/src/ShellCheck/Formatter/Diff.hs @@ -38,9 +38,6 @@ import System.FilePath import Test.QuickCheck -import Debug.Trace -ltt x = trace (show x) x - format :: FormatterOptions -> IO Formatter format options = do foundIssues <- newIORef False diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 30bbb69..e269b4d 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -37,7 +37,7 @@ import Data.Functor import Data.List (isPrefixOf, isInfixOf, isSuffixOf, partition, sortBy, intercalate, nub, find) import Data.Maybe import Data.Monoid -import Debug.Trace +import Debug.Trace -- STRIP import GHC.Exts (sortWith) import Prelude hiding (readList) import System.IO @@ -3372,13 +3372,13 @@ parsesCleanly parser string = runIdentity $ do -- For printf debugging: print the value of an expression -- Example: return $ dump $ T_Literal id [c] -dump :: Show a => a -> a -dump x = trace (show x) x +dump :: Show a => a -> a -- STRIP +dump x = trace (show x) x -- STRIP -- Like above, but print a specific expression: -- Example: return $ dumps ("Returning: " ++ [c]) $ T_Literal id [c] -dumps :: Show x => x -> a -> a -dumps t = trace (show t) +dumps :: Show x => x -> a -> a -- STRIP +dumps t = trace (show t) -- STRIP parseWithNotes parser = do item <- parser diff --git a/striptests b/striptests index c68c16a..6e64607 100755 --- a/striptests +++ b/striptests @@ -29,6 +29,7 @@ detestify() { state = 0; } + /STRIP/ { next; } /LANGUAGE TemplateHaskell/ { next; } /^import.*Test\./ { next; } From db4701d8b54fa1bb980bd24cfe2d1fe446278c6b Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 18 Sep 2021 19:32:12 -0700 Subject: [PATCH 192/459] Add a `setgitversion` script to update the version string with git --- .github/workflows/build.yml | 3 +++ setgitversion | 11 +++++++++++ src/ShellCheck/Data.hs | 2 +- 3 files changed, 15 insertions(+), 1 deletion(-) create mode 100755 setgitversion diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0acece2..e28a43f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,9 +16,12 @@ jobs: - name: Checkout repository uses: actions/checkout@v2 + with: + fetch-depth: 0 - name: Package Source run: | + ./setgitversion mkdir source cabal sdist mv dist-newstyle/sdist/*.tar.gz source/source.tar.gz diff --git a/setgitversion b/setgitversion new file mode 100755 index 0000000..3afad61 --- /dev/null +++ b/setgitversion @@ -0,0 +1,11 @@ +#!/bin/sh -xe +# This script hardcodes the `git describe` version as ShellCheck's version number. +# This is done to allow shellcheck --version to differ from the cabal version when +# building git snapshots. + +file="src/ShellCheck/Data.hs" +test -e "$file" +tmp=$(mktemp) +version=$(git describe) +sed -e "s/=.*VERSIONSTRING.*/= \"$version\" -- VERSIONSTRING, DO NOT SUBMIT/" "$file" > "$tmp" +mv "$tmp" "$file" diff --git a/src/ShellCheck/Data.hs b/src/ShellCheck/Data.hs index fb4a1e4..793a4de 100644 --- a/src/ShellCheck/Data.hs +++ b/src/ShellCheck/Data.hs @@ -4,7 +4,7 @@ import ShellCheck.Interface import Data.Version (showVersion) import Paths_ShellCheck (version) -shellcheckVersion = showVersion version +shellcheckVersion = showVersion version -- VERSIONSTRING internalVariables = [ -- Generic From 3a296cd788cdb9313ed2fc2aa5e86a876269d407 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 19 Sep 2021 12:27:16 -0700 Subject: [PATCH 193/459] The removed check was SC1004, not SC1003 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fef7680..48fbfeb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,7 +29,7 @@ - Leading `_` can now be used to suppress warnings about unused variables ### Removed -- SC1003: Literal backslash+linefeed in '' was found to be usually correct +- SC1004: Literal backslash+linefeed in '' was found to be usually correct ## v0.7.2 - 2021-04-19 From ad92cb411291631b5f1d06df083b92f02128b33d Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 25 Sep 2021 19:46:27 -0700 Subject: [PATCH 194/459] Disable UUOC for cat with unquoted variable (fixes #2333) --- src/ShellCheck/ASTLib.hs | 13 ++++++------- src/ShellCheck/Analytics.hs | 6 ++++-- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 20e5be4..5c13697 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -287,14 +287,14 @@ isArrayExpansion (T_DollarBraced _ _ l) = isArrayExpansion _ = False -- Is it possible that this arg becomes multiple args? -mayBecomeMultipleArgs t = willBecomeMultipleArgs t || f t +mayBecomeMultipleArgs t = willBecomeMultipleArgs t || f False t where - f (T_DollarBraced _ _ l) = + f quoted (T_DollarBraced _ _ l) = let string = concat $ oversimplify l in - "!" `isPrefixOf` string - f (T_DoubleQuoted _ parts) = any f parts - f (T_NormalWord _ parts) = any f parts - f _ = False + not quoted || "!" `isPrefixOf` string + f quoted (T_DoubleQuoted _ parts) = any (f True) parts + f quoted (T_NormalWord _ parts) = any (f quoted) parts + f _ _ = False -- Is it certain that this word will becomes multiple words? willBecomeMultipleArgs t = willConcatInAssignment t || f t @@ -302,7 +302,6 @@ willBecomeMultipleArgs t = willConcatInAssignment t || f t f T_Extglob {} = True f T_Glob {} = True f T_BraceExpansion {} = True - f (T_DoubleQuoted _ parts) = any f parts f (T_NormalWord _ parts) = any f parts f _ = False diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index de59be1..749a93d 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -506,7 +506,9 @@ checkWrongArithmeticAssignment _ _ = return () prop_checkUuoc1 = verify checkUuoc "cat foo | grep bar" prop_checkUuoc2 = verifyNot checkUuoc "cat * | grep bar" -prop_checkUuoc3 = verify checkUuoc "cat $var | grep bar" +prop_checkUuoc3 = verify checkUuoc "cat \"$var\" | grep bar" +prop_checkUuoc3b = verifyNot checkUuoc "cat $var | grep bar" +prop_checkUuoc3c = verifyNot checkUuoc "cat \"${!var}\" | grep bar" prop_checkUuoc4 = verifyNot checkUuoc "cat $var" prop_checkUuoc5 = verifyNot checkUuoc "cat \"$@\"" prop_checkUuoc6 = verifyNot checkUuoc "cat -n | grep bar" @@ -659,7 +661,7 @@ prop_checkForInQuoted7 = verify checkForInQuoted "for f in ls, grep, mv; do true prop_checkForInQuoted8 = verify checkForInQuoted "for f in 'ls', 'grep', 'mv'; do true; done" prop_checkForInQuoted9 = verifyNot checkForInQuoted "for f in 'ls,' 'grep,' 'mv'; do true; done" checkForInQuoted _ (T_ForIn _ f [T_NormalWord _ [word@(T_DoubleQuoted id list)]] _) - | any (\x -> willSplit x && not (mayBecomeMultipleArgs x)) list + | any willSplit list && not (mayBecomeMultipleArgs word) || maybe False wouldHaveBeenGlob (getLiteralString word) = err id 2066 "Since you double quoted this, it will not word split, and the loop will only run once." checkForInQuoted _ (T_ForIn _ f [T_NormalWord _ [T_SingleQuoted id _]] _) = From fac97a5301b39306e6f26cb6d4fadbbaa4e6f8b0 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 25 Sep 2021 20:23:58 -0700 Subject: [PATCH 195/459] Don't emit SC2140 when trapped string is /, = or : (fixes #2334) --- src/ShellCheck/Analytics.hs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 749a93d..bb95cf4 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1766,6 +1766,8 @@ prop_checkInexplicablyUnquoted7 = verifyNot checkInexplicablyUnquoted "${dir/\"f prop_checkInexplicablyUnquoted8 = verifyNot checkInexplicablyUnquoted " 'foo'\\\n 'bar'" prop_checkInexplicablyUnquoted9 = verifyNot checkInexplicablyUnquoted "[[ $x =~ \"foo\"(\"bar\"|\"baz\") ]]" prop_checkInexplicablyUnquoted10 = verifyNot checkInexplicablyUnquoted "cmd ${x+--name=\"$x\" --output=\"$x.out\"}" +prop_checkInexplicablyUnquoted11 = verifyNot checkInexplicablyUnquoted "echo \"foo\"/\"bar\"" +prop_checkInexplicablyUnquoted12 = verifyNot checkInexplicablyUnquoted "declare \"foo\"=\"bar\"" checkInexplicablyUnquoted params (T_NormalWord id tokens) = mapM_ check (tails tokens) where check (T_SingleQuoted _ _:T_Literal id str:_) @@ -1777,7 +1779,10 @@ checkInexplicablyUnquoted params (T_NormalWord id tokens) = mapM_ check (tails t T_DollarExpansion id _ -> warnAboutExpansion id T_DollarBraced id _ _ -> warnAboutExpansion id T_Literal id s - | not (quotesSingleThing a && quotesSingleThing b || isSpecial (getPath (parentMap params) trapped)) -> + | not (quotesSingleThing a && quotesSingleThing b + || s `elem` ["=", ":", "/"] + || isSpecial (getPath (parentMap params) trapped) + ) -> warnAboutLiteral id _ -> return () From 093df8cb2448a3b2f352b4e3105d2af45ccf3206 Mon Sep 17 00:00:00 2001 From: Christian Nassif-Haynes Date: Mon, 6 Sep 2021 05:52:34 +1000 Subject: [PATCH 196/459] Add extra checks for masked return codes --- src/ShellCheck/Analytics.hs | 109 ++++++++++++++++++++++++++++++ src/ShellCheck/AnalyzerLib.hs | 18 +++++ src/ShellCheck/Checks/Commands.hs | 2 - src/ShellCheck/Data.hs | 2 + 4 files changed, 129 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index de59be1..a09e804 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -260,6 +260,13 @@ optionalTreeChecks = [ cdPositive = "set -e; func() { cp *.txt ~/backup; rm *.txt; }; func && echo ok", cdNegative = "set -e; func() { cp *.txt ~/backup; rm *.txt; }; func; echo ok" }, checkSetESuppressed) + + ,(newCheckDescription { + cdName = "check-extra-masked-returns", + cdDescription = "Check for additional cases where exit codes are masked", + cdPositive = "rm -r \"$(get_chroot_dir)/home\"", + cdNegative = "set -e; dir=\"$(get_chroot_dir)\"; rm -r \"$dir/home\"" + }, checkExtraMaskedReturns) ] optionalCheckMap :: Map.Map String (Parameters -> Token -> [TokenComment]) @@ -4734,5 +4741,107 @@ checkSetESuppressed params t = isIn t cmds = getId t `elem` map getId cmds +prop_checkExtraMaskedReturns1 = verifyTree checkExtraMaskedReturns "cat < <(ls)" +prop_checkExtraMaskedReturns2 = verifyTree checkExtraMaskedReturns "read -ra arr <(ls)" +prop_checkExtraMaskedReturns3 = verifyTree checkExtraMaskedReturns "ls >(cat)" +prop_checkExtraMaskedReturns4 = verifyTree checkExtraMaskedReturns "false | true" +prop_checkExtraMaskedReturns5 = verifyNotTree checkExtraMaskedReturns "set -o pipefail; false | true" +prop_checkExtraMaskedReturns6 = verifyNotTree checkExtraMaskedReturns "false | true || true" +prop_checkExtraMaskedReturns7 = verifyTree checkExtraMaskedReturns "true $(false)" +prop_checkExtraMaskedReturns8 = verifyTree checkExtraMaskedReturns "x=$(false)$(true)" +prop_checkExtraMaskedReturns9 = verifyNotTree checkExtraMaskedReturns "x=$(false)true" +prop_checkExtraMaskedReturns10 = verifyTree checkExtraMaskedReturns "x=`false``false`" +prop_checkExtraMaskedReturns11 = verifyTree checkExtraMaskedReturns "x=\"$(false)$(true)\"" +prop_checkExtraMaskedReturns12 = verifyTree checkExtraMaskedReturns "x=\"$(false)\"\"$(true)\"" +prop_checkExtraMaskedReturns13 = verifyTree checkExtraMaskedReturns "true <<<$(false)" +prop_checkExtraMaskedReturns14 = verifyNotTree checkExtraMaskedReturns "echo asdf | false" +prop_checkExtraMaskedReturns15 = verifyNotTree checkExtraMaskedReturns "readonly x=$(false)" +prop_checkExtraMaskedReturns16 = verifyTree checkExtraMaskedReturns "readarray -t files < <(ls)" +prop_checkExtraMaskedReturns17 = verifyNotTree checkExtraMaskedReturns "x=( $(false) false )" +prop_checkExtraMaskedReturns18 = verifyTree checkExtraMaskedReturns "x=( $(false) $(false) )" +prop_checkExtraMaskedReturns19 = verifyNotTree checkExtraMaskedReturns "x=( $(false) [4]=false )" +prop_checkExtraMaskedReturns20 = verifyTree checkExtraMaskedReturns "x=( $(false) [4]=$(false) )" +prop_checkExtraMaskedReturns21 = verifyTree checkExtraMaskedReturns "cat << foo\n $(false)\nfoo" +prop_checkExtraMaskedReturns22 = verifyTree checkExtraMaskedReturns "[[ $(false) ]]" +prop_checkExtraMaskedReturns23 = verifyNotTree checkExtraMaskedReturns "x=$(false) y=z" +prop_checkExtraMaskedReturns24 = verifyNotTree checkExtraMaskedReturns "x=$(( $(date +%s) ))" +prop_checkExtraMaskedReturns25 = verifyTree checkExtraMaskedReturns "echo $(( $(date +%s) ))" +prop_checkExtraMaskedReturns26 = verifyNotTree checkExtraMaskedReturns "x=( $(false) )" +prop_checkExtraMaskedReturns27 = verifyTree checkExtraMaskedReturns "x=$(false) false" +prop_checkExtraMaskedReturns28 = verifyTree checkExtraMaskedReturns "x=$(false) y=$(false)" +prop_checkExtraMaskedReturns29 = verifyNotTree checkExtraMaskedReturns "false < <(set -e)" +prop_checkExtraMaskedReturns30 = verifyNotTree checkExtraMaskedReturns "false < <(shopt -s cdspell)" +prop_checkExtraMaskedReturns31 = verifyNotTree checkExtraMaskedReturns "false < <(dirname \"${BASH_SOURCE[0]}\")" +prop_checkExtraMaskedReturns32 = verifyNotTree checkExtraMaskedReturns "false < <(basename \"${BASH_SOURCE[0]}\")" +prop_checkExtraMaskedReturns33 = verifyNotTree checkExtraMaskedReturns "{ false || true; } | true" +prop_checkExtraMaskedReturns34 = verifyNotTree checkExtraMaskedReturns "{ false || :; } | true" +checkExtraMaskedReturns params t = runNodeAnalysis findMaskingNodes params t + where + findMaskingNodes _ (T_Arithmetic _ list) = findMaskedNodesInList [list] + findMaskingNodes _ (T_Array _ list) = findMaskedNodesInList $ allButLastSimpleCommands list + findMaskingNodes _ (T_Condition _ _ condition) = findMaskedNodesInList [condition] + findMaskingNodes _ (T_DoubleQuoted _ list) = findMaskedNodesInList $ allButLastSimpleCommands list + findMaskingNodes _ (T_HereDoc _ _ _ _ list) = findMaskedNodesInList list + findMaskingNodes _ (T_HereString _ word) = findMaskedNodesInList [word] + findMaskingNodes _ (T_NormalWord _ parts) = findMaskedNodesInList $ allButLastSimpleCommands parts + findMaskingNodes _ (T_Pipeline _ _ cmds) | not (hasPipefail params) = findMaskedNodesInList $ allButLastSimpleCommands cmds + findMaskingNodes _ (T_ProcSub _ _ list) = findMaskedNodesInList list + findMaskingNodes _ (T_SimpleCommand _ assigns (_:args)) = findMaskedNodesInList $ assigns ++ args + findMaskingNodes _ (T_SimpleCommand _ assigns []) = findMaskedNodesInList $ allButLastSimpleCommands assigns + findMaskingNodes _ _ = return () + + findMaskedNodesInList = mapM_ (doAnalysis findMaskedNodes) + + isMaskedNode t = not (isHarmlessCommand t || isCheckedElsewhere t || isMaskDeliberate t) + findMaskedNodes t@(T_SimpleCommand _ _ (_:_)) = when (isMaskedNode t) $ inform t + findMaskedNodes t@T_Condition {} = when (isMaskedNode t) $ inform t + findMaskedNodes _ = return () + + containsSimpleCommand t = isNothing $ doAnalysis go t + where + go t = case t of + T_SimpleCommand {} -> fail "" + _ -> return () + + allButLastSimpleCommands cmds = + if null simpleCommands then [] else init simpleCommands + where + simpleCommands = filter containsSimpleCommand cmds + + inform t = info (getId t) 2312 ("Consider invoking this command " + ++ "separately to avoid masking its return value (or use '|| true' " + ++ "to ignore).") + + isMaskDeliberate t = hasParent isOrIf t + where + isOrIf _ (T_OrIf _ _ (T_Pipeline _ _ [T_Redirecting _ _ cmd])) + = getCommandBasename cmd `elem` [Just "true", Just ":"] + isOrIf _ _ = False + + isCheckedElsewhere t = hasParent isDeclaringCommand t + where + isDeclaringCommand t _ = fromMaybe False $ do + basename <- getCommandBasename t + return $ basename `elem` declaringCommands + + isHarmlessCommand t = fromMaybe False $ do + basename <- getCommandBasename t + return $ basename `elem` [ + "echo" + ,"basename" + ,"dirname" + ,"printf" + ,"set" + ,"shopt" + ] + + parentChildPairs t = go $ parents params t + where + go (child:parent:rest) = (parent, child):go (parent:rest) + go _ = [] + + hasParent pred t = any (uncurry pred) (parentChildPairs t) + + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 9cb6cd2..be54fbb 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -83,6 +83,8 @@ data Parameters = Parameters { hasInheritErrexit :: Bool, -- Whether this script has 'set -e' anywhere. hasSetE :: Bool, + -- Whether this script has 'set -o pipefail' anywhere. + hasPipefail :: Bool, -- A linear (bad) analysis of data flow variableFlow :: [StackData], -- A map from Id to parent Token @@ -204,6 +206,12 @@ makeParameters spec = Dash -> True Sh -> True Ksh -> False, + hasPipefail = + case shellType params of + Bash -> containsPipefail root + Dash -> True + Sh -> True + Ksh -> containsPipefail root, shellTypeSpecified = isJust (asShellType spec) || isJust (asFallbackShell spec), parentMap = getParentTree root, variableFlow = getVariableFlow params root, @@ -226,6 +234,16 @@ containsSetE root = isNothing $ doAnalysis (guard . not . isSetE) root _ -> False re = mkRegex "[[:space:]]-[^-]*e" +containsPipefail root = isNothing $ doAnalysis (guard . not . isPipefail) root + where + isPipefail t = + case t of + T_SimpleCommand {} -> + t `isUnqualifiedCommand` "set" && + ("pipefail" `elem` oversimplify t || + "o" `elem` map snd (getAllFlags t)) + _ -> False + containsShopt shopt root = isNothing $ doAnalysis (guard . not . isShoptLastPipe) root where diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index d0ada73..77bdbf6 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -101,8 +101,6 @@ commandChecks = [ ++ map checkArgComparison declaringCommands ++ map checkMaskedReturns declaringCommands -declaringCommands = ["local", "declare", "export", "readonly", "typeset", "let"] - optionalChecks = map fst optionalCommandChecks optionalCommandChecks :: [(CheckDescription, CommandCheck)] diff --git a/src/ShellCheck/Data.hs b/src/ShellCheck/Data.hs index 793a4de..e22b424 100644 --- a/src/ShellCheck/Data.hs +++ b/src/ShellCheck/Data.hs @@ -138,3 +138,5 @@ shellForExecutable name = _ -> Nothing flagsForRead = "sreu:n:N:i:p:a:t:" + +declaringCommands = ["local", "declare", "export", "readonly", "typeset", "let"] From 6f7eee4a2779ced87334c4962b64241f83fdf7e0 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 2 Oct 2021 12:58:28 -0700 Subject: [PATCH 197/459] Mention check-extra-masked-returns in changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 48fbfeb..f7be144 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,8 @@ - `disable=all` now conveniently disables all warnings - `external-sources=true` directive can be added to .shellcheckrc to make shellcheck behave as if `-x` was specified. +- Optional `check-extra-masked-returns` for pointing out commands with + suppressed exit codes (SC2312). - SC2286-SC2288: Warn when command name ends in a symbol like `/.)'"` - SC2289: Warn when command name contains tabs or linefeeds - SC2291: Warn about repeated unquoted spaces between words in echo From 38251abe26d952e4602d819c74fcdba46af4b066 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 7 Oct 2021 17:14:41 -0700 Subject: [PATCH 198/459] Add suggestion level in text for TTY output (fixes #2339) --- CHANGELOG.md | 1 + src/ShellCheck/Formatter/TTY.hs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f7be144..1741552 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ - SC2181 now only triggers on single condition tests like `[ $? = 0 ]`. - Quote warnings are now emitted for declaration utilities in sh - Leading `_` can now be used to suppress warnings about unused variables +- TTY output now includes warning level in text as well as color ### Removed - SC1004: Literal backslash+linefeed in '' was found to be usually correct diff --git a/src/ShellCheck/Formatter/TTY.hs b/src/ShellCheck/Formatter/TTY.hs index bb57894..8dd90d4 100644 --- a/src/ShellCheck/Formatter/TTY.hs +++ b/src/ShellCheck/Formatter/TTY.hs @@ -174,7 +174,7 @@ showFixedString color comments lineNum fileLines = cuteIndent :: PositionedComment -> String cuteIndent comment = replicate (fromIntegral $ colNo comment - 1) ' ' ++ - makeArrow ++ " " ++ code (codeNo comment) ++ ": " ++ messageText comment + makeArrow ++ " " ++ code (codeNo comment) ++ " (" ++ severityText comment ++ "): " ++ messageText comment where arrow n = '^' : replicate (fromIntegral $ n-2) '-' ++ "^" makeArrow = From 05bdeae3abc539e27246f761f8184f1df7ee9878 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 7 Oct 2021 17:26:08 -0700 Subject: [PATCH 199/459] Mention require-double-brackets in CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1741552..c98254f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ shellcheck behave as if `-x` was specified. - Optional `check-extra-masked-returns` for pointing out commands with suppressed exit codes (SC2312). +- Optional `require-double-brackets` for recommending \[\[ ]] (SC2292). - SC2286-SC2288: Warn when command name ends in a symbol like `/.)'"` - SC2289: Warn when command name contains tabs or linefeeds - SC2291: Warn about repeated unquoted spaces between words in echo From 205ba429b3d8afee08e6e9f5ff504fc8aa62a9c8 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 7 Oct 2021 18:50:44 -0700 Subject: [PATCH 200/459] Warn about `read foo[i]` expanding as glob (fixes #2345) --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 9 +++++++-- src/ShellCheck/Checks/Commands.hs | 18 ++++++++++++++++-- 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c98254f..a1456a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ - SC2304-SC2306: Warn about unquoted globs in expr arguments - SC2307: Warn about insufficient number of arguments to expr - SC2308: Suggest other approaches for non-standard expr extensions +- SC2313: Warn about `read` with unquoted, array indexed variable ### Fixed - SC2102 about repetitions in ranges no longer triggers on [[ -v arr[xx] ]] diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 3c99cea..787a501 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2689,12 +2689,13 @@ prop_checkCharRangeGlob1 = verify checkCharRangeGlob "ls *[:digit:].jpg" prop_checkCharRangeGlob2 = verifyNot checkCharRangeGlob "ls *[[:digit:]].jpg" prop_checkCharRangeGlob3 = verify checkCharRangeGlob "ls [10-15]" prop_checkCharRangeGlob4 = verifyNot checkCharRangeGlob "ls [a-zA-Z]" -prop_checkCharRangeGlob5 = verifyNot checkCharRangeGlob "tr -d [a-zA-Z]" -- tr has 2060 +prop_checkCharRangeGlob5 = verifyNot checkCharRangeGlob "tr -d [aa]" -- tr has 2060 prop_checkCharRangeGlob6 = verifyNot checkCharRangeGlob "[[ $x == [!!]* ]]" prop_checkCharRangeGlob7 = verifyNot checkCharRangeGlob "[[ -v arr[keykey] ]]" prop_checkCharRangeGlob8 = verifyNot checkCharRangeGlob "[[ arr[keykey] -gt 1 ]]" +prop_checkCharRangeGlob9 = verifyNot checkCharRangeGlob "read arr[keykey]" -- tr has 2313 checkCharRangeGlob p t@(T_Glob id str) | - isCharClass str && not (isParamTo (parentMap p) "tr" t) && not (isDereferenced t) = + isCharClass str && not isIgnoredCommand && not (isDereferenced t) = if ":" `isPrefixOf` contents && ":" `isSuffixOf` contents && contents /= ":" @@ -2712,6 +2713,10 @@ checkCharRangeGlob p t@(T_Glob id str) | '^':rest -> rest x -> x + isIgnoredCommand = fromMaybe False $ do + cmd <- getClosestCommand (parentMap p) t + return $ isCommandMatch cmd (`elem` ["tr", "read"]) + -- Check if this is a dereferencing context like [[ -v array[operandhere] ]] isDereferenced = fromMaybe False . msum . map isDereferencingOp . getPath (parentMap p) isDereferencingOp t = diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 77bdbf6..519ba4f 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -790,6 +790,7 @@ prop_checkReadExpansions5 = verify checkReadExpansions "read \"$var\"" prop_checkReadExpansions6 = verify checkReadExpansions "read -a $var" prop_checkReadExpansions7 = verifyNot checkReadExpansions "read $1" prop_checkReadExpansions8 = verifyNot checkReadExpansions "read ${var?}" +prop_checkReadExpansions9 = verify checkReadExpansions "read arr[val]" checkReadExpansions = CommandCheck (Exactly "read") check where options = getGnuOpts flagsForRead @@ -797,13 +798,26 @@ checkReadExpansions = CommandCheck (Exactly "read") check opts <- options $ arguments cmd return [y | (x,(_, y)) <- opts, null x || x == "a"] - check cmd = mapM_ warning $ getVars cmd - warning t = sequence_ $ do + check cmd = do + mapM_ dollarWarning $ getVars cmd + mapM_ arrayWarning $ arguments cmd + + dollarWarning t = sequence_ $ do name <- getSingleUnmodifiedBracedString t guard $ isVariableName name -- e.g. not $1 return . warn (getId t) 2229 $ "This does not read '" ++ name ++ "'. Remove $/${} for that, or use ${var?} to quiet." + arrayWarning word = + when (any isUnquotedBracket $ getWordParts word) $ + warn (getId word) 2313 $ + "Quote array indices to avoid them expanding as globs." + + isUnquotedBracket t = + case t of + T_Glob _ ('[':_) -> True + _ -> False + -- Return the single variable expansion that makes up this word, if any. -- e.g. $foo -> $foo, "$foo"'' -> $foo , "hello $name" -> Nothing getSingleUnmodifiedBracedString :: Token -> Maybe String From 3aedda766de75b21b6b6eff58da3881c41e4fa16 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 9 Oct 2021 11:40:52 -0700 Subject: [PATCH 201/459] For `while getopts; do case ..` checks, make sure variable matches --- src/ShellCheck/Checks/Commands.hs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 519ba4f..edc41ed 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -959,15 +959,24 @@ prop_checkWhileGetoptsCase2 = verify checkWhileGetoptsCase "while getopts 'a:' x prop_checkWhileGetoptsCase3 = verifyNot checkWhileGetoptsCase "while getopts 'a:b' x; do case $x in a) foo;; b) bar;; *) :;esac; done" prop_checkWhileGetoptsCase4 = verifyNot checkWhileGetoptsCase "while getopts 'a:123' x; do case $x in a) foo;; [0-9]) bar;; esac; done" prop_checkWhileGetoptsCase5 = verifyNot checkWhileGetoptsCase "while getopts 'a:' x; do case $x in a) foo;; \\?) bar;; *) baz;; esac; done" +prop_checkWhileGetoptsCase6 = verifyNot checkWhileGetoptsCase "while getopts 'a:b' x; do case $y in a) foo;; esac; done" +prop_checkWhileGetoptsCase7 = verifyNot checkWhileGetoptsCase "while getopts 'a:b' x; do case x$x in xa) foo;; xb) foo;; esac; done" checkWhileGetoptsCase = CommandCheck (Exactly "getopts") f where f :: Token -> Analysis - f t@(T_SimpleCommand _ _ (cmd:arg1:_)) = do + f t@(T_SimpleCommand _ _ (cmd:arg1:name:_)) = do path <- getPathM t sequence_ $ do options <- getLiteralString arg1 + getoptsVar <- getLiteralString name (T_WhileExpression _ _ body) <- findFirst whileLoop path - caseCmd <- mapMaybe findCase body !!! 0 + caseCmd@(T_CaseExpression _ var _) <- mapMaybe findCase body !!! 0 + + -- Make sure getopts name and case variable matches + [T_DollarBraced _ _ bracedWord] <- return $ getWordParts var + [T_Literal _ caseVar] <- return $ getWordParts bracedWord + guard $ caseVar == getoptsVar + return $ check (getId arg1) (map (:[]) $ filter (/= ':') options) caseCmd f _ = return () From c3aaa27540e1ab0e93074937176d8bbe89554afb Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 9 Oct 2021 12:13:41 -0700 Subject: [PATCH 202/459] Skip SC2214 if variable is modified in loop (fixes #2351) --- src/ShellCheck/Analytics.hs | 10 +--------- src/ShellCheck/AnalyzerLib.hs | 17 +++++++++++++++++ src/ShellCheck/Checks/Commands.hs | 5 +++++ 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 787a501..912fab5 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1957,17 +1957,9 @@ subshellAssignmentCheck params t = findSubshelled [] _ _ = return () findSubshelled (Assignment x@(_, _, str, data_):rest) scopes@((reason,scope):restscope) deadVars = - if isTrueAssignment data_ + if isTrueAssignmentSource data_ then findSubshelled rest ((reason, x:scope):restscope) $ Map.insert str Alive deadVars else findSubshelled rest scopes deadVars - where - isTrueAssignment c = - case c of - DataString SourceChecked -> False - DataString SourceDeclaration -> False - DataArray SourceChecked -> False - DataArray SourceDeclaration -> False - _ -> True findSubshelled (Reference (_, readToken, str):rest) scopes deadVars = do unless (shouldIgnore str) $ case Map.findWithDefault Alive str deadVars of diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index be54fbb..e36a14f 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -1005,6 +1005,23 @@ isUnmodifiedParameterExpansion t = in getBracedReference str == str _ -> False +isTrueAssignmentSource c = + case c of + DataString SourceChecked -> False + DataString SourceDeclaration -> False + DataArray SourceChecked -> False + DataArray SourceDeclaration -> False + _ -> True + +modifiesVariable params token name = + or $ map check flow + where + flow = getVariableFlow params token + check t = + case t of + Assignment (_, _, n, source) -> isTrueAssignmentSource source && n == name + _ -> False + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index edc41ed..e80ed3a 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -961,11 +961,13 @@ prop_checkWhileGetoptsCase4 = verifyNot checkWhileGetoptsCase "while getopts 'a: prop_checkWhileGetoptsCase5 = verifyNot checkWhileGetoptsCase "while getopts 'a:' x; do case $x in a) foo;; \\?) bar;; *) baz;; esac; done" prop_checkWhileGetoptsCase6 = verifyNot checkWhileGetoptsCase "while getopts 'a:b' x; do case $y in a) foo;; esac; done" prop_checkWhileGetoptsCase7 = verifyNot checkWhileGetoptsCase "while getopts 'a:b' x; do case x$x in xa) foo;; xb) foo;; esac; done" +prop_checkWhileGetoptsCase8 = verifyNot checkWhileGetoptsCase "while getopts 'a:b' x; do x=a; case $x in a) foo;; esac; done" checkWhileGetoptsCase = CommandCheck (Exactly "getopts") f where f :: Token -> Analysis f t@(T_SimpleCommand _ _ (cmd:arg1:name:_)) = do path <- getPathM t + params <- ask sequence_ $ do options <- getLiteralString arg1 getoptsVar <- getLiteralString name @@ -977,6 +979,9 @@ checkWhileGetoptsCase = CommandCheck (Exactly "getopts") f [T_Literal _ caseVar] <- return $ getWordParts bracedWord guard $ caseVar == getoptsVar + -- Make sure the variable isn't modified + guard . not $ modifiesVariable params (T_BraceGroup (Id 0) body) getoptsVar + return $ check (getId arg1) (map (:[]) $ filter (/= ':') options) caseCmd f _ = return () From 0d128dd918be22b764d1625393775eb4080ca006 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Fri, 15 Oct 2021 12:06:33 -0700 Subject: [PATCH 203/459] Mention known incompatibilities in man page --- shellcheck.1.md | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/shellcheck.1.md b/shellcheck.1.md index 1103acc..b1f0c25 100644 --- a/shellcheck.1.md +++ b/shellcheck.1.md @@ -335,10 +335,32 @@ locales where encoding is unspecified (such as the `C` locale). Windows users seeing `commitBuffer: invalid argument (invalid character)` should set their terminal to use UTF-8 with `chcp 65001`. -# AUTHORS +# KNOWN INCOMPATIBILITIES -ShellCheck is developed and maintained by Vidar Holen, with assistance from a -long list of wonderful contributors. +(If nothing in this section makes sense, you are unlikely to be affected by it) + +To avoid confusing and misguided suggestions, ShellCheck requires function +bodies to be either `{ brace groups; }` or `( subshells )`, and function names +containing `[]*=!` are only recognized after a `function` keyword. + +The following unconventional function definitions are identical in Bash, +but ShellCheck only recognizes the latter. + + [x!=y] () [[ $1 ]] + function [x!=y] () { [[ $1 ]]; } + +Shells without the `function` keyword do not allow these characters in function +names to begin with. Function names containing `{}` are not supported at all. + +Further, if ShellCheck sees `[x!=y]` it will assume this is an invalid +comparison. To invoke the above function, quote the command as in `'[x!=y]'`, +or to retain the same globbing behavior, use `command [x!=y]`. + +ShellCheck imposes additional restrictions on the `[` command to help diagnose +common invalid uses. While `[ $x= 1 ]` is defined in POSIX, ShellCheck will +assume it was intended as the much more likely comparison `[ "$x" = 1 ]` and +fail accordingly. For unconventional or dynamic uses of the `[` command, use +`test` or `\[` instead. # REPORTING BUGS @@ -346,6 +368,11 @@ Bugs and issues can be reported on GitHub: https://github.com/koalaman/shellcheck/issues +# AUTHORS + +ShellCheck is developed and maintained by Vidar Holen, with assistance from a +long list of wonderful contributors. + # COPYRIGHT Copyright 2012-2019, Vidar Holen and contributors. From 788aee1b7c1c74863c85837fbd53dd45cf810d14 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Fri, 15 Oct 2021 14:39:30 -0700 Subject: [PATCH 204/459] Treat typeset similar to declare (fixes #2354) --- src/ShellCheck/Analytics.hs | 1 + src/ShellCheck/AnalyzerLib.hs | 21 ++++++++++++++------- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 912fab5..8960d93 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2380,6 +2380,7 @@ prop_checkUnused46= verifyTree checkUnusedAssignments "readonly foo=(bar)" prop_checkUnused47= verifyNotTree checkUnusedAssignments "a=1; alias hello='echo $a'" prop_checkUnused48= verifyNotTree checkUnusedAssignments "_a=1" prop_checkUnused49= verifyNotTree checkUnusedAssignments "declare -A array; key=a; [[ -v array[$key] ]]" +prop_checkUnused50= verifyNotTree checkUnusedAssignments "foofunc() { :; }; typeset -fx foofunc" checkUnusedAssignments params t = execWriter (mapM_ warnFor unused) where diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index e36a14f..311364f 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -571,14 +571,12 @@ isClosingFileOp op = -- Consider 'export/declare -x' a reference, since it makes the var available getReferencedVariableCommand base@(T_SimpleCommand _ _ (T_NormalWord _ (T_Literal _ x:_):rest)) = case x of + "declare" -> forDeclare + "typeset" -> forDeclare + "export" -> if "f" `elem` flags then [] else concatMap getReference rest - "declare" -> if - any (`elem` flags) ["x", "p"] && - (not $ any (`elem` flags) ["f", "F"]) - then concatMap getReference rest - else [] "local" -> if "x" `elem` flags then concatMap getReference rest else [] @@ -589,6 +587,13 @@ getReferencedVariableCommand base@(T_SimpleCommand _ _ (T_NormalWord _ (T_Litera "alias" -> [(base, token, name) | token <- rest, name <- getVariablesFromLiteralToken token] _ -> [] where + forDeclare = + if + any (`elem` flags) ["x", "p"] && + (not $ any (`elem` flags) ["f", "F"]) + then concatMap getReference rest + else [] + getReference t@(T_Assignment _ _ name _ value) = [(t, t, name)] getReference t@(T_NormalWord _ [T_Literal _ name]) | not ("-" `isPrefixOf` name) = [(t, t, name)] getReference _ = [] @@ -628,8 +633,8 @@ getModifiedVariableCommand base@(T_SimpleCommand id cmdPrefix (T_NormalWord _ (T "export" -> if "f" `elem` flags then [] else concatMap getModifierParamString rest - "declare" -> if any (`elem` flags) ["F", "f", "p"] then [] else declaredVars - "typeset" -> declaredVars + "declare" -> forDeclare + "typeset" -> forDeclare "local" -> concatMap getModifierParamString rest "readonly" -> @@ -661,6 +666,8 @@ getModifiedVariableCommand base@(T_SimpleCommand id cmdPrefix (T_NormalWord _ (T T_NormalWord id1 [T_DoubleQuoted id2 [T_Literal id3 (stripEquals s)]] stripEqualsFrom t = t + forDeclare = if any (`elem` flags) ["F", "f", "p"] then [] else declaredVars + declaredVars = concatMap (getModifierParam defaultType) rest where defaultType = if any (`elem` flags) ["a", "A"] then DataArray else DataString From 7b2092b3cd89c98067e31cd68d2738f8afa2b0ad Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Fri, 15 Oct 2021 15:29:52 -0700 Subject: [PATCH 205/459] Give more examples of what ShellCheck looks for --- README.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d91f962..6f3e4a9 100644 --- a/README.md +++ b/README.md @@ -358,6 +358,7 @@ echo 'Don't forget to restart!' # Singlequote closed by apostrophe echo 'Don\'t try this at home' # Attempting to escape ' in '' echo 'Path is $PATH' # Variables in single quotes trap "echo Took ${SECONDS}s" 0 # Prematurely expanded trap +unset var[i] # Array index treated as glob ``` ### Conditionals @@ -376,6 +377,7 @@ ShellCheck can recognize many types of incorrect test statements. [ grep -q foo file ] # Command without $(..) [[ "$$file" == *.jpg ]] # Comparisons that can't succeed (( 1 -lt 2 )) # Using test operators in ((..)) +[ x ] & [ y ] | [ z ] # Accidental backgrounding and piping ``` ### Frequently misused commands @@ -447,6 +449,8 @@ echo "Hello $name" # Unassigned lowercase variables cmd | read bar; echo $bar # Assignments in subshells cat foo | cp bar # Piping to commands that don't read printf '%s: %s\n' foo # Mismatches in printf argument count +eval "${array[@]}" # Lost word boundaries in array eval +for i in "${x[@]}"; do ${x[$i]} # Using array value as key ``` ### Robustness @@ -471,6 +475,7 @@ ShellCheck will warn when using features not supported by the shebang. For examp echo {1..$n} # Works in ksh, but not bash/dash/sh echo {1..10} # Works in ksh and bash, but not dash/sh echo -n 42 # Works in ksh, bash and dash, undefined in sh +expr match str regex # Unportable alias for `expr str : regex` trap 'exit 42' sigint # Unportable signal spec cmd &> file # Unportable redirection operator read foo < /dev/tcp/host/22 # Unportable intercepted files @@ -491,10 +496,15 @@ rm “file” # Unicode quotes echo "Hello world" # Carriage return / DOS line endings echo hello \ # Trailing spaces after \ var=42 echo $var # Expansion of inlined environment -#!/bin/bash -x -e # Common shebang errors +!# bin/bash -x -e # Common shebang errors echo $((n/180*100)) # Unnecessary loss of precision ls *[:digit:].txt # Bad character class globs sed 's/foo/bar/' file > file # Redirecting to input +var2=$var2 # Variable assigned to itself +[ x$var = xval ] # Antiquated x-comparisons +ls() { ls -l "$@"; } # Infinitely recursive wrapper +alias ls='ls -l'; ls foo # Alias used before it takes effect +for x; do for x; do # Nested loop uses same variable while getopts "a" f; do case $f in "b") # Unhandled getopts flags ``` From 290fc8b945243bda1e867694c5abd5ecfdf2bc18 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Fri, 15 Oct 2021 18:03:05 -0700 Subject: [PATCH 206/459] Have quickscripts search for relevant paths (fixes #2286) --- quickrun | 10 +++++++++- quicktest | 11 ++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/quickrun b/quickrun index 172ae88..e0e0547 100755 --- a/quickrun +++ b/quickrun @@ -2,4 +2,12 @@ # quickrun runs ShellCheck in an interpreted mode. # This allows testing changes without recompiling. -runghc -isrc -idist/build/autogen shellcheck.hs "$@" +path=$(find . -type f -path './dist*/Paths_ShellCheck.hs' | sort | head -n 1) +if [ -z "$path" ] +then + echo >&2 "Unable to find Paths_ShellCheck.hs. Please 'cabal build' once." + exit 1 +fi +path="${path%/*}" + +exec runghc -isrc -i"$path" shellcheck.hs "$@" diff --git a/quicktest b/quicktest index 55041a7..6a1cf61 100755 --- a/quicktest +++ b/quicktest @@ -3,8 +3,17 @@ # This allows running tests without compiling, which can be faster. # 'cabal test' remains the source of truth. +path=$(find . -type f -path './dist*/Paths_ShellCheck.hs' | sort | head -n 1) +if [ -z "$path" ] +then + echo >&2 "Unable to find Paths_ShellCheck.hs. Please 'cabal build' once." + exit 1 +fi +path="${path%/*}" + + ( -var=$(echo 'main' | ghci test/shellcheck.hs 2>&1 | tee /dev/stderr) +var=$(echo 'main' | ghci -isrc -i"$path" test/shellcheck.hs 2>&1 | tee /dev/stderr) if [[ $var == *ExitSuccess* ]] then exit 0 From 0dd5c67bdf0fe5f0d03e4d2f6fc04c3796408ccd Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 21 Oct 2021 20:58:14 -0700 Subject: [PATCH 207/459] Warn about [^..] in Dash (fixes #2361) --- src/ShellCheck/Checks/ShellSupport.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 1d373e5..092c2bf 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -180,7 +180,7 @@ prop_checkBashisms95 = verify checkBashisms "#!/bin/sh\necho $_" prop_checkBashisms96 = verifyNot checkBashisms "#!/bin/dash\necho $_" prop_checkBashisms97 = verify checkBashisms "#!/bin/sh\necho ${var,}" prop_checkBashisms98 = verify checkBashisms "#!/bin/sh\necho ${var^^}" -prop_checkBashisms99 = verifyNot checkBashisms "#!/bin/dash\necho [^f]oo" +prop_checkBashisms99 = verify checkBashisms "#!/bin/dash\necho [^f]oo" checkBashisms = ForShell [Sh, Dash] $ \t -> do params <- ask kludge params t @@ -235,7 +235,7 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do where file = onlyLiteralString word isNetworked = any (`isPrefixOf` file) ["/dev/tcp", "/dev/udp"] - bashism (T_Glob id str) | not isDash && "[^" `isInfixOf` str = + bashism (T_Glob id str) | "[^" `isInfixOf` str = warnMsg id 3026 "^ in place of ! in glob bracket expressions is" bashism t@(TA_Variable id str _) | isBashVariable str = From efd49e486f47f7e10f9b01a8afd8f116eb0893c5 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 30 Oct 2021 17:47:30 -0700 Subject: [PATCH 208/459] Consider all forms of TA_Assignment to remove spaces (fixes #2364) --- src/ShellCheck/Analytics.hs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 8960d93..64ba8a8 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2048,6 +2048,7 @@ prop_checkSpacefulness42= verifyNotTree checkSpacefulness "run $1 --flags" prop_checkSpacefulness43= verifyNotTree checkSpacefulness "$foo=42" prop_checkSpacefulness44= verifyTree checkSpacefulness "#!/bin/sh\nexport var=$value" prop_checkSpacefulness45= verifyNotTree checkSpacefulness "wait -zzx -p foo; echo $foo" +prop_checkSpacefulness46= verifyNotTree checkSpacefulness "x=0; (( x += 1 )); echo $x" data SpaceStatus = SpaceSome | SpaceNone | SpaceEmpty deriving (Eq) instance Semigroup SpaceStatus where @@ -2138,6 +2139,7 @@ checkSpacefulness' onFind params t = where emit x = tell [x] + writeF _ (TA_Assignment {}) name _ = setSpaces name SpaceNone >> return [] writeF _ _ name (DataString SourceExternal) = setSpaces name SpaceSome >> return [] writeF _ _ name (DataString SourceInteger) = setSpaces name SpaceNone >> return [] From 8db220ae43280860f8ec920eea1e66608803d0f9 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 6 Nov 2021 15:37:59 -0700 Subject: [PATCH 209/459] Include `local -r` in check-extra-masked-returns (fixes #2362) --- src/ShellCheck/Analytics.hs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 64ba8a8..244236f 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -4782,6 +4782,8 @@ prop_checkExtraMaskedReturns31 = verifyNotTree checkExtraMaskedReturns "false < prop_checkExtraMaskedReturns32 = verifyNotTree checkExtraMaskedReturns "false < <(basename \"${BASH_SOURCE[0]}\")" prop_checkExtraMaskedReturns33 = verifyNotTree checkExtraMaskedReturns "{ false || true; } | true" prop_checkExtraMaskedReturns34 = verifyNotTree checkExtraMaskedReturns "{ false || :; } | true" +prop_checkExtraMaskedReturns35 = verifyTree checkExtraMaskedReturns "f() { local -r x=$(false); }" + checkExtraMaskedReturns params t = runNodeAnalysis findMaskingNodes params t where findMaskingNodes _ (T_Arithmetic _ list) = findMaskedNodesInList [list] @@ -4828,8 +4830,13 @@ checkExtraMaskedReturns params t = runNodeAnalysis findMaskingNodes params t isCheckedElsewhere t = hasParent isDeclaringCommand t where isDeclaringCommand t _ = fromMaybe False $ do - basename <- getCommandBasename t - return $ basename `elem` declaringCommands + cmd <- getCommand t + basename <- getCommandBasename cmd + return $ + case basename of + -- local -r x=$(false) is intentionally ignored for SC2155 + "local" | "r" `elem` (map snd $ getAllFlags cmd) -> False + _ -> basename `elem` declaringCommands isHarmlessCommand t = fromMaybe False $ do basename <- getCommandBasename t From bcca66eb6b7c18c58ac668d7ab47bb1890b489e3 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 6 Nov 2021 15:46:19 -0700 Subject: [PATCH 210/459] Update release checklist --- test/check_release | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/check_release b/test/check_release index 91001fb..fd1dbca 100755 --- a/test/check_release +++ b/test/check_release @@ -56,19 +56,19 @@ cat << EOF Manual Checklist $((i++)). Make sure none of the automated checks above failed -$((i++)). Make sure Travis build currently passes: https://travis-ci.org/koalaman/shellcheck +$((i++)). Make sure GitHub Build currently passes: https://github.com/koalaman/shellcheck/actions $((i++)). Make sure SnapCraft build currently works: https://build.snapcraft.io/user/koalaman $((i++)). Run test/distrotest to ensure that most distros can build OOTB. $((i++)). Format and read over the manual for bad formatting and outdated info. -$((i++)). Make sure the Hackage package builds, so that all files are +$((i++)). Make sure the Hackage package builds. Release Steps $((j++)). \`cabal sdist\` to generate a Hackage package $((j++)). \`git push --follow-tags\` to push commit -$((j++)). Wait for Travis to build +$((j++)). Wait for GitHub Actions to build. $((j++)). Verify release: - a. Check that the new versions are uploaded: https://shellcheck.storage.googleapis.com/index.html + a. Check that the new versions are uploaded: https://github.com/koalaman/shellcheck/tags b. Check that the docker images have version tags: https://hub.docker.com/u/koalaman $((j++)). If no disaster, upload to Hackage: http://hackage.haskell.org/upload $((j++)). Push a new commit that updates CHANGELOG.md From 71f1db660937d4d83e2709079eea444137bef4c3 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 6 Nov 2021 18:21:11 -0700 Subject: [PATCH 211/459] Update distro tests --- test/distrotest | 10 +++++----- test/stacktest | 5 +++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/test/distrotest b/test/distrotest index 346a706..50a5a17 100755 --- a/test/distrotest +++ b/test/distrotest @@ -64,13 +64,13 @@ ubuntu:latest apt-get update && apt-get install -y cabal-install haskell:latest true opensuse/leap:latest zypper install -y cabal-install ghc fedora:latest dnf install -y cabal-install ghc-template-haskell-devel findutils -archlinux/base:latest pacman -S -y --noconfirm cabal-install ghc-static base-devel +archlinux:latest pacman -S -y --noconfirm cabal-install ghc-static base-devel -# Other versions we want to support -ubuntu:18.04 apt-get update && apt-get install -y cabal-install +# Ubuntu LTS +ubuntu:20.04 apt-get update && apt-get install -y cabal-install -# Misc Haskell including current and latest Stack build -ubuntu:18.04 set -e; apt-get update && apt-get install -y curl && curl -sSL https://get.haskellstack.org/ | sh -s - -f && cd /mnt && exec test/stacktest +# Stack on Ubuntu LTS +ubuntu:20.04 set -e; apt-get update && apt-get install -y curl && curl -sSL https://get.haskellstack.org/ | sh -s - -f && cd /mnt && exec test/stacktest EOF exit "$final" diff --git a/test/stacktest b/test/stacktest index dc0113f..ae04f1b 100755 --- a/test/stacktest +++ b/test/stacktest @@ -18,10 +18,11 @@ command -v stack || stack setup || die "Failed to setup with default resolver" stack build --test || die "Failed to build/test with default resolver" +# Nice to haves, but not necessary for resolver in "${resolvers[@]}" do - stack --resolver="$resolver" setup || die "Failed to setup $resolver" - stack --resolver="$resolver" build --test || die "Failed build/test with $resolver!" + stack --resolver="$resolver" setup || die "Failed to setup $resolver. This probably doesn't matter." + stack --resolver="$resolver" build --test || die "Failed build/test with $resolver! This probably doesn't matter." done echo "Success" From 14a38b94cc2ca6810492a457d64f7019b059eed6 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 6 Nov 2021 18:59:24 -0700 Subject: [PATCH 212/459] Update stack resolver --- stack.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stack.yaml b/stack.yaml index 6dee632..4cf5c74 100644 --- a/stack.yaml +++ b/stack.yaml @@ -2,7 +2,7 @@ # For more information, see: https://docs.haskellstack.org/en/stable/yaml_configuration/ # Specifies the GHC version and set of packages available (e.g., lts-3.5, nightly-2015-09-21, ghc-7.10.2) -resolver: lts-13.26 +resolver: lts-18.15 # Local packages, usually specified by relative directory name packages: From 3b6972fbf14d98dded7ebfc65af6b73724be4efa Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 6 Nov 2021 19:07:34 -0700 Subject: [PATCH 213/459] Update copyright years --- shellcheck.1.md | 2 +- src/ShellCheck/ASTLib.hs | 2 +- src/ShellCheck/Analytics.hs | 2 +- src/ShellCheck/AnalyzerLib.hs | 2 +- src/ShellCheck/Checker.hs | 2 +- src/ShellCheck/Checks/Commands.hs | 2 +- src/ShellCheck/Checks/ShellSupport.hs | 2 +- src/ShellCheck/Parser.hs | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/shellcheck.1.md b/shellcheck.1.md index b1f0c25..146d791 100644 --- a/shellcheck.1.md +++ b/shellcheck.1.md @@ -375,7 +375,7 @@ long list of wonderful contributors. # COPYRIGHT -Copyright 2012-2019, Vidar Holen and contributors. +Copyright 2012-2021, Vidar Holen and contributors. Licensed under the GNU General Public License version 3 or later, see https://gnu.org/licenses/gpl.html diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 5c13697..83ba5f8 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -1,5 +1,5 @@ {- - Copyright 2012-2019 Vidar Holen + Copyright 2012-2021 Vidar Holen This file is part of ShellCheck. https://www.shellcheck.net diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 244236f..652c2fb 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1,5 +1,5 @@ {- - Copyright 2012-2019 Vidar Holen + Copyright 2012-2021 Vidar Holen This file is part of ShellCheck. https://www.shellcheck.net diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 311364f..687859f 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -1,5 +1,5 @@ {- - Copyright 2012-2019 Vidar Holen + Copyright 2012-2021 Vidar Holen This file is part of ShellCheck. https://www.shellcheck.net diff --git a/src/ShellCheck/Checker.hs b/src/ShellCheck/Checker.hs index cce1063..ef8182f 100644 --- a/src/ShellCheck/Checker.hs +++ b/src/ShellCheck/Checker.hs @@ -1,5 +1,5 @@ {- - Copyright 2012-2019 Vidar Holen + Copyright 2012-2020 Vidar Holen This file is part of ShellCheck. https://www.shellcheck.net diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index e80ed3a..5a29a26 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -1,5 +1,5 @@ {- - Copyright 2012-2019 Vidar Holen + Copyright 2012-2021 Vidar Holen This file is part of ShellCheck. https://www.shellcheck.net diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 092c2bf..22a6a5f 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -1,5 +1,5 @@ {- - Copyright 2012-2016 Vidar Holen + Copyright 2012-2020 Vidar Holen This file is part of ShellCheck. https://www.shellcheck.net diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index e269b4d..92eb61f 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -1,5 +1,5 @@ {- - Copyright 2012-2019 Vidar Holen + Copyright 2012-2021 Vidar Holen This file is part of ShellCheck. https://www.shellcheck.net From eea823e3d088c0bcf048dae9ad9c321be09560ad Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 6 Nov 2021 22:05:19 -0700 Subject: [PATCH 214/459] Fix bad version on stable releases --- .github/workflows/build.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e28a43f..5595219 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,22 +19,22 @@ jobs: with: fetch-depth: 0 - - name: Package Source - run: | - ./setgitversion - mkdir source - cabal sdist - mv dist-newstyle/sdist/*.tar.gz source/source.tar.gz - - name: Deduce tags run: | - exec > source/tags - echo "latest" + mkdir source + echo "latest" > source/tags if tag=$(git describe --exact-match --tags) then - echo "stable" - echo "$tag" + echo "stable" >> source/tags + echo "$tag" >> source/tags fi + cat source/tags + + - name: Package Source + run: | + grep "stable" source/tags || ./setgitversion + cabal sdist + mv dist-newstyle/sdist/*.tar.gz source/source.tar.gz - name: Upload artifact uses: actions/upload-artifact@v2 From e5ad4cf420a7f7b8e5eaac872b14a1619051cf10 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 6 Nov 2021 19:08:58 -0700 Subject: [PATCH 215/459] Stable version 0.8.0 This release is dedicated to dibblego, who pushed me down the Haskell rabbit hole. In 2006 I thought you were crazy. Today I *know* you are. --- CHANGELOG.md | 2 +- ShellCheck.cabal | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a1456a0..9118671 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## Git (0.8.0) +## v0.8.0 - 2021-11-06 ### Added - `disable=all` now conveniently disables all warnings - `external-sources=true` directive can be added to .shellcheckrc to make diff --git a/ShellCheck.cabal b/ShellCheck.cabal index 9433f55..1167c82 100644 --- a/ShellCheck.cabal +++ b/ShellCheck.cabal @@ -1,5 +1,5 @@ Name: ShellCheck -Version: 0.7.2 +Version: 0.8.0 Synopsis: Shell script analysis tool License: GPL-3 License-file: LICENSE From 4c186c20b9a5a3a95768d9a609c93df37991c663 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 6 Nov 2021 23:18:19 -0700 Subject: [PATCH 216/459] Post-release CHANGELOG update --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9118671..f65bbfa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +## Git +### Added + +### Fixed + +### Changed + + ## v0.8.0 - 2021-11-06 ### Added - `disable=all` now conveniently disables all warnings From c5de58ae84954b1c773bfe2b3710daf0d7e92fbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Skytt=C3=A4?= Date: Sat, 13 Nov 2021 12:50:53 +0200 Subject: [PATCH 217/459] Comment spelling fixes --- src/ShellCheck/AnalyzerLib.hs | 2 +- src/ShellCheck/Checks/Commands.hs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 687859f..5f77fba 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -351,7 +351,7 @@ isQuoteFreeNode strict shell tree t = T_SelectIn {} -> return (not strict) _ -> Nothing - -- Check whether this assigment is self-quoting due to being a recognized + -- Check whether this assignment is self-quoting due to being a recognized -- assignment passed to a Declaration Utility. This will soon be required -- by POSIX: https://austingroupbugs.net/view.php?id=351 assignmentIsQuoting t = shellParsesParamsAsAssignments || not (isAssignmentParamToCommand t) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 5a29a26..1a48a28 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -735,7 +735,7 @@ getPrintfFormats = getFormats -- \____ _____/\___ ____/ \____ ____/\_________ _________/ \______ / -- V V V V V -- flags field width precision format character rest - -- field width and precision can be specified with a '*' instead of a digit, + -- field width and precision can be specified with an '*' instead of a digit, -- in which case printf will accept one more argument for each '*' used From d9a9d5db86122d608b1a6929609e799d175e6aa6 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 14 Nov 2021 16:39:32 -0800 Subject: [PATCH 218/459] Mark prefix/postfix inc/dec as integers (fixes #2376) --- src/ShellCheck/Analytics.hs | 3 ++- src/ShellCheck/AnalyzerLib.hs | 8 +++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 652c2fb..188d683 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2049,6 +2049,8 @@ prop_checkSpacefulness43= verifyNotTree checkSpacefulness "$foo=42" prop_checkSpacefulness44= verifyTree checkSpacefulness "#!/bin/sh\nexport var=$value" prop_checkSpacefulness45= verifyNotTree checkSpacefulness "wait -zzx -p foo; echo $foo" prop_checkSpacefulness46= verifyNotTree checkSpacefulness "x=0; (( x += 1 )); echo $x" +prop_checkSpacefulness47= verifyNotTree checkSpacefulness "x=0; (( x-- )); echo $x" +prop_checkSpacefulness48= verifyNotTree checkSpacefulness "x=0; (( ++x )); echo $x" data SpaceStatus = SpaceSome | SpaceNone | SpaceEmpty deriving (Eq) instance Semigroup SpaceStatus where @@ -2139,7 +2141,6 @@ checkSpacefulness' onFind params t = where emit x = tell [x] - writeF _ (TA_Assignment {}) name _ = setSpaces name SpaceNone >> return [] writeF _ _ name (DataString SourceExternal) = setSpaces name SpaceSome >> return [] writeF _ _ name (DataString SourceInteger) = setSpaces name SpaceNone >> return [] diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 687859f..9b53f9f 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -511,13 +511,11 @@ getModifiedVariables t = T_SimpleCommand {} -> getModifiedVariableCommand t - TA_Unary _ "++|" v@(TA_Variable _ name _) -> - [(t, v, name, DataString $ SourceFrom [v])] - TA_Unary _ "|++" v@(TA_Variable _ name _) -> - [(t, v, name, DataString $ SourceFrom [v])] + TA_Unary _ op v@(TA_Variable _ name _) | "--" `isInfixOf` op || "++" `isInfixOf` op -> + [(t, v, name, DataString SourceInteger)] TA_Assignment _ op (TA_Variable _ name _) rhs -> do guard $ op `elem` ["=", "*=", "/=", "%=", "+=", "-=", "<<=", ">>=", "&=", "^=", "|="] - return (t, t, name, DataString $ SourceFrom [rhs]) + return (t, t, name, DataString SourceInteger) T_BatsTest {} -> [ (t, t, "lines", DataArray SourceExternal), From 499c99372eaef411fc5223393c6799005eebc085 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 14 Nov 2021 21:34:21 -0800 Subject: [PATCH 219/459] Rewrite SC2032 warning and mention line number (fixes #2353) --- src/ShellCheck/Analytics.hs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 188d683..4a7f364 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2297,7 +2297,7 @@ checkFunctionsUsedExternally params t = let args = skipOver t argv let argStrings = map (\x -> (fromMaybe "" $ getLiteralString x, x)) args let candidates = getPotentialCommands name argStrings - mapM_ (checkArg name) candidates + mapM_ (checkArg name (getId t)) candidates _ -> return () checkCommand _ _ = return () @@ -2323,14 +2323,19 @@ checkFunctionsUsedExternally params t = functionsAndAliases = Map.union (functions t) (aliases t) - checkArg cmd (_, arg) = sequence_ $ do + patternContext id = + case posLine . fst <$> Map.lookup id (tokenPositions params) of + Just l -> " on line " <> show l <> "." + _ -> "." + + checkArg cmd cmdId (_, arg) = sequence_ $ do literalArg <- getUnquotedLiteral arg -- only consider unquoted literals definitionId <- Map.lookup literalArg functionsAndAliases return $ do warn (getId arg) 2033 - "Shell functions can't be passed to external commands." + "Shell functions can't be passed to external commands. Use separate script or sh -c." info definitionId 2032 $ - "Use own script or sh -c '..' to run this from " ++ cmd ++ "." + "This function can't be invoked via " ++ cmd ++ patternContext cmdId prop_checkUnused0 = verifyNotTree checkUnusedAssignments "var=foo; echo $var" prop_checkUnused1 = verifyTree checkUnusedAssignments "var=foo; echo $bar" From 9092080a84af694f1a1fbc1d59f89d296f362069 Mon Sep 17 00:00:00 2001 From: Martin Schulze Date: Mon, 15 Nov 2021 11:49:36 +0100 Subject: [PATCH 220/459] bats: Add check for useless negation (SC2314/15) --- src/ShellCheck/Analytics.hs | 43 +++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 4a7f364..7c31137 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -199,6 +199,7 @@ nodeChecks = [ ,checkComparisonWithLeadingX ,checkCommandWithTrailingSymbol ,checkUnquotedParameterExpansionPattern + ,checkBatsTestDoesNotUseNegation ] optionalChecks = map fst optionalTreeChecks @@ -4863,5 +4864,47 @@ checkExtraMaskedReturns params t = runNodeAnalysis findMaskingNodes params t hasParent pred t = any (uncurry pred) (parentChildPairs t) +-- hard error on negated command that is not last +prop_checkBatsTestDoesNotUseNegation1 = verify checkBatsTestDoesNotUseNegation "#!/usr/bin/env/bats\n@test \"name\" { ! true; false; }" +prop_checkBatsTestDoesNotUseNegation2 = verify checkBatsTestDoesNotUseNegation "#!/usr/bin/env/bats\n@test \"name\" { ! [[ -e test ]]; false; }" +prop_checkBatsTestDoesNotUseNegation3 = verify checkBatsTestDoesNotUseNegation "#!/usr/bin/env/bats\n@test \"name\" { ! [ -e test ]; false; }" +-- acceptable formats: +-- using run +prop_checkBatsTestDoesNotUseNegation4 = verifyNot checkBatsTestDoesNotUseNegation "#!/usr/bin/env/bats\n@test \"name\" { run ! true; }" +-- using || false +prop_checkBatsTestDoesNotUseNegation5 = verifyNot checkBatsTestDoesNotUseNegation "#!/usr/bin/env/bats\n@test \"name\" { ! [[ -e test ]] || false; }" +prop_checkBatsTestDoesNotUseNegation6 = verifyNot checkBatsTestDoesNotUseNegation "#!/usr/bin/env/bats\n@test \"name\" { ! [ -e test ] || false; }" +-- only style warning when last command +prop_checkBatsTestDoesNotUseNegation7 = verifyCodes checkBatsTestDoesNotUseNegation [2314] "#!/usr/bin/env/bats\n@test \"name\" { ! true; }" +prop_checkBatsTestDoesNotUseNegation8 = verifyCodes checkBatsTestDoesNotUseNegation [2315] "#!/usr/bin/env/bats\n@test \"name\" { ! [[ -e test ]]; }" +prop_checkBatsTestDoesNotUseNegation9 = verifyCodes checkBatsTestDoesNotUseNegation [2315] "#!/usr/bin/env/bats\n@test \"name\" { ! [ -e test ]; }" + +checkBatsTestDoesNotUseNegation params t = + case t of + T_BatsTest _ _ (T_BraceGroup _ commands) -> mapM_ (check commands) commands + _ -> return () + where + check commands t = + case t of + T_Banged id (T_Pipeline _ _ [T_Redirecting _ _ (T_Condition idCondition _ _)]) -> + if t `isLastOf` commands + then style id 2315 "In Bats, ! will not fail the test if it is not the last command anymore. Fold the `!` into the conditional!" + else err id 2315 + "In Bats, ! does not cause a test failure. Fold the `!` into the conditional!" + + T_Banged id cmd -> if t `isLastOf` commands + then styleWithFix id 2314 + "In Bats, ! will not fail the test if it is not the last command anymore. Use `run ! ` (on Bats >= 1.5.0) instead." + (fixWith [replaceStart id params 0 "run "]) + else errWithFix id 2314 + "In Bats, ! does not cause a test failure. Use 'run ! ' (on Bats >= 1.5.0) instead." + (fixWith [replaceStart id params 0 "run "]) + _ -> return () + isLastOf t commands = + case commands of + [x] -> x == t + x:rest -> isLastOf t rest + [] -> False + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) From d7971dafd1539de37bb29cb554d5fd4f4b95553b Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 4 Dec 2021 17:37:12 -0800 Subject: [PATCH 221/459] Minor formatting fixes --- src/ShellCheck/Analytics.hs | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 7c31137..d48104e 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -4868,7 +4868,7 @@ checkExtraMaskedReturns params t = runNodeAnalysis findMaskingNodes params t prop_checkBatsTestDoesNotUseNegation1 = verify checkBatsTestDoesNotUseNegation "#!/usr/bin/env/bats\n@test \"name\" { ! true; false; }" prop_checkBatsTestDoesNotUseNegation2 = verify checkBatsTestDoesNotUseNegation "#!/usr/bin/env/bats\n@test \"name\" { ! [[ -e test ]]; false; }" prop_checkBatsTestDoesNotUseNegation3 = verify checkBatsTestDoesNotUseNegation "#!/usr/bin/env/bats\n@test \"name\" { ! [ -e test ]; false; }" --- acceptable formats: +-- acceptable formats: -- using run prop_checkBatsTestDoesNotUseNegation4 = verifyNot checkBatsTestDoesNotUseNegation "#!/usr/bin/env/bats\n@test \"name\" { run ! true; }" -- using || false @@ -4886,21 +4886,18 @@ checkBatsTestDoesNotUseNegation params t = where check commands t = case t of - T_Banged id (T_Pipeline _ _ [T_Redirecting _ _ (T_Condition idCondition _ _)]) -> + T_Banged id (T_Pipeline _ _ [T_Redirecting _ _ (T_Condition idCondition _ _)]) -> if t `isLastOf` commands then style id 2315 "In Bats, ! will not fail the test if it is not the last command anymore. Fold the `!` into the conditional!" - else err id 2315 - "In Bats, ! does not cause a test failure. Fold the `!` into the conditional!" - + else err id 2315 "In Bats, ! does not cause a test failure. Fold the `!` into the conditional!" + T_Banged id cmd -> if t `isLastOf` commands - then styleWithFix id 2314 - "In Bats, ! will not fail the test if it is not the last command anymore. Use `run ! ` (on Bats >= 1.5.0) instead." + then styleWithFix id 2314 "In Bats, ! will not fail the test if it is not the last command anymore. Use `run ! ` (on Bats >= 1.5.0) instead." (fixWith [replaceStart id params 0 "run "]) - else errWithFix id 2314 - "In Bats, ! does not cause a test failure. Use 'run ! ' (on Bats >= 1.5.0) instead." + else errWithFix id 2314 "In Bats, ! does not cause a test failure. Use 'run ! ' (on Bats >= 1.5.0) instead." (fixWith [replaceStart id params 0 "run "]) _ -> return () - isLastOf t commands = + isLastOf t commands = case commands of [x] -> x == t x:rest -> isLastOf t rest From 3a118246ef2e6e37129cb0f1c0d1dae319385520 Mon Sep 17 00:00:00 2001 From: Rune Juhl Jacobsen Date: Tue, 14 Dec 2021 16:00:47 +0100 Subject: [PATCH 222/459] Fix bug in 2126 when using after/before flags with grep Using `--after-context`/`-A` or `--before-context`/`-B` would give a warning recommending the user to use `grep -c`, even though that would give a different result than using `grep | wc -l`: ```fundamental $ echo -e "1\n2\n3" | grep -cA 3 1 1 $ echo -e "1\n2\n3" | grep -A 3 1 | wc -l 3 ``` --- src/ShellCheck/Analytics.hs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index d48104e..e6e47ea 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -545,6 +545,10 @@ prop_checkPipePitfalls15 = verifyNot checkPipePitfalls "foo | grep bar | wc -cmw prop_checkPipePitfalls16 = verifyNot checkPipePitfalls "foo | grep -r bar | wc -l" prop_checkPipePitfalls17 = verifyNot checkPipePitfalls "foo | grep -l bar | wc -l" prop_checkPipePitfalls18 = verifyNot checkPipePitfalls "foo | grep -L bar | wc -l" +prop_checkPipePitfalls19 = verifyNot checkPipePitfalls "foo | grep -A2 bar | wc -l" +prop_checkPipePitfalls20 = verifyNot checkPipePitfalls "foo | grep -B999 bar | wc -l" +prop_checkPipePitfalls21 = verifyNot checkPipePitfalls "foo | grep --after-context 999 bar | wc -l" +prop_checkPipePitfalls22 = verifyNot checkPipePitfalls "foo | grep -B 1 --after-context 999 bar | wc -l" checkPipePitfalls _ (T_Pipeline id _ commands) = do for ["find", "xargs"] $ \(find:xargs:_) -> @@ -566,10 +570,10 @@ checkPipePitfalls _ (T_Pipeline id _ commands) = do let flagsGrep = maybe [] (map snd . getAllFlags) $ getCommand grep flagsWc = maybe [] (map snd . getAllFlags) $ getCommand wc in - unless (any (`elem` ["l", "files-with-matches", "L", "files-without-matches", "o", "only-matching", "r", "R", "recursive"]) flagsGrep + unless (any (`elem` ["l", "files-with-matches", "L", "files-without-matches", "o", "only-matching", "r", "R", "recursive", "A", "after-context", "B", "before-context"]) flagsGrep || any (`elem` ["m", "chars", "w", "words", "c", "bytes", "L", "max-line-length"]) flagsWc || null flagsWc) $ - style (getId grep) 2126 "Consider using grep -c instead of grep|wc -l." + style (getId grep) 2126 "Consider using 'grep -c' instead of 'grep|wc -l'." didLs <- fmap or . sequence $ [ for' ["ls", "grep"] $ From e6e558946ca70985dfe1ac905af19e615cbc4cc6 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 21 Dec 2021 14:07:46 -0800 Subject: [PATCH 223/459] Improve decoding of single quoted literals (fixes #2418) --- src/ShellCheck/ASTLib.hs | 35 ++++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 83ba5f8..7c88432 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -369,6 +369,21 @@ getGlobOrLiteralString = getLiteralStringExt f f (T_Glob _ str) = return str f _ = Nothing + +prop_getLiteralString1 = getLiteralString (T_DollarSingleQuoted (Id 0) "\\x01") == Just "\1" +prop_getLiteralString2 = getLiteralString (T_DollarSingleQuoted (Id 0) "\\xyz") == Just "\\xyz" +prop_getLiteralString3 = getLiteralString (T_DollarSingleQuoted (Id 0) "\\x1") == Just "\x1" +prop_getLiteralString4 = getLiteralString (T_DollarSingleQuoted (Id 0) "\\x1y") == Just "\x1y" +prop_getLiteralString5 = getLiteralString (T_DollarSingleQuoted (Id 0) "\\xy") == Just "\\xy" +prop_getLiteralString6 = getLiteralString (T_DollarSingleQuoted (Id 0) "\\x") == Just "\\x" +prop_getLiteralString7 = getLiteralString (T_DollarSingleQuoted (Id 0) "\\1x") == Just "\1x" +prop_getLiteralString8 = getLiteralString (T_DollarSingleQuoted (Id 0) "\\12x") == Just "\o12x" +prop_getLiteralString9 = getLiteralString (T_DollarSingleQuoted (Id 0) "\\123x") == Just "\o123x" +prop_getLiteralString10 = getLiteralString (T_DollarSingleQuoted (Id 0) "\\1234") == Just "\o123\&4" +prop_getLiteralString11 = getLiteralString (T_DollarSingleQuoted (Id 0) "\\1") == Just "\1" +prop_getLiteralString12 = getLiteralString (T_DollarSingleQuoted (Id 0) "\\12") == Just "\o12" +prop_getLiteralString13 = getLiteralString (T_DollarSingleQuoted (Id 0) "\\123") == Just "\o123" + -- Maybe get the literal value of a token, using a custom function -- to map unrecognized Tokens into strings. getLiteralStringExt :: Monad m => (Token -> m String) -> Token -> m String @@ -401,14 +416,15 @@ getLiteralStringExt more = g '\\' -> '\\' : rest 'x' -> case cs of - (x:y:more) -> - if isHexDigit x && isHexDigit y - then chr (16*(digitToInt x) + (digitToInt y)) : rest - else '\\':c:rest + (x:y:more) | isHexDigit x && isHexDigit y -> + chr (16*(digitToInt x) + (digitToInt y)) : decodeEscapes more + (x:more) | isHexDigit x -> + chr (digitToInt x) : decodeEscapes more + more -> '\\' : 'x' : decodeEscapes more _ | isOctDigit c -> - let digits = take 3 $ takeWhile isOctDigit (c:cs) - num = parseOct digits - in (if num < 256 then chr num else '?') : rest + let (digits, more) = spanMax isOctDigit 3 (c:cs) + num = (parseOct digits) `mod` 256 + in (chr num) : decodeEscapes more _ -> '\\' : c : rest where rest = decodeEscapes cs @@ -416,6 +432,11 @@ getLiteralStringExt more = g where f n "" = n f n (c:rest) = f (n * 8 + digitToInt c) rest + spanMax f n list = + let (first, second) = span f list + (prefix, suffix) = splitAt n first + in + (prefix, suffix ++ second) decodeEscapes (c:cs) = c : decodeEscapes cs decodeEscapes [] = [] From ade2bf7b871362c9c30878294aef8ab775f3ea07 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 9 Jan 2022 16:50:50 -0800 Subject: [PATCH 224/459] Allow parsing [[ x = ["$y"] ]] (fixes #2165) --- src/ShellCheck/Parser.hs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 92eb61f..8fbdb5a 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -556,7 +556,7 @@ readConditionContents single = notFollowedBy2 (try (spacing >> string "]")) x <- readNormalWord pos <- getPosition - when (endedWith "]" x && notArrayIndex x) $ do + when (notArrayIndex x && endedWith "]" x && not (x `containsLiteral` "[")) $ do parseProblemAt pos ErrorC 1020 $ "You need a space before the " ++ (if single then "]" else "]]") ++ "." fail "Missing space before ]" @@ -572,6 +572,7 @@ readConditionContents single = endedWith _ _ = False notArrayIndex (T_NormalWord id s@(_:T_Literal _ t:_)) = t /= "[" notArrayIndex _ = True + containsLiteral x s = s `isInfixOf` onlyLiteralString x readCondAndOp = readAndOrOp TC_And "&&" False <|> readAndOrOp TC_And "-a" True @@ -941,6 +942,9 @@ prop_readCondition23 = isOk readCondition "[[ -v arr[$var] ]]" prop_readCondition25 = isOk readCondition "[[ lex.yy.c -ot program.l ]]" prop_readCondition26 = isOk readScript "[[ foo ]]\\\n && bar" prop_readCondition27 = not $ isOk readConditionCommand "[[ x ]] foo" +prop_readCondition28 = isOk readCondition "[[ x = [\"$1\"] ]]" +prop_readCondition29 = isOk readCondition "[[ x = [*] ]]" + readCondition = called "test expression" $ do opos <- getPosition start <- startSpan From 2292e852e5be3a727a667dce9def79c1d12f0804 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 23 Jan 2022 14:23:56 -0800 Subject: [PATCH 225/459] Switch linux-x86_64 build from Ubuntu to Alpine for musl --- build/linux.x86_64/Dockerfile | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/build/linux.x86_64/Dockerfile b/build/linux.x86_64/Dockerfile index f0ad16a..3112ac2 100644 --- a/build/linux.x86_64/Dockerfile +++ b/build/linux.x86_64/Dockerfile @@ -1,16 +1,10 @@ -FROM ubuntu:20.04 +FROM alpine:latest ENV TARGETNAME linux.x86_64 # Install GHC and cabal USER root -ENV DEBIAN_FRONTEND noninteractive -RUN apt-get update && apt-get install -y ghc curl xz-utils - -# So we'd like a later version of Cabal that supports --enable-executable-static, -# but we can't use Ubuntu 20.10 because coreutils has switched to new syscalls that -# the TravisCI kernel doesn't support. Download it manually. -RUN curl "https://downloads.haskell.org/~cabal/cabal-install-3.2.0.0/cabal-install-3.2.0.0-x86_64-unknown-linux.tar.xz" | tar xJv -C /usr/bin +RUN apk add ghc cabal g++ libffi-dev curl bash # Use ld.bfd instead of ld.gold due to # x86_64-linux-gnu/libpthread.a(pthread_cond_init.o)(.note.stapsdt+0x14): error: From 88cdb4e2c9b45becb21bd02cd7b205d5bef8cb56 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 3 Feb 2022 19:23:46 -0800 Subject: [PATCH 226/459] Warn about spaces around = in alias (fixes #2442) --- src/ShellCheck/Checks/Commands.hs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 1a48a28..72d8c09 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -98,7 +98,7 @@ commandChecks = [ ,checkUnquotedEchoSpaces ,checkEvalArray ] - ++ map checkArgComparison declaringCommands + ++ map checkArgComparison ("alias" : declaringCommands) ++ map checkMaskedReturns declaringCommands @@ -1253,6 +1253,7 @@ prop_checkArgComparison3 = verifyNot (checkArgComparison "declare") "declare a=b prop_checkArgComparison4 = verify (checkArgComparison "export") "export a +=b" prop_checkArgComparison7 = verifyNot (checkArgComparison "declare") "declare -a +i foo" prop_checkArgComparison8 = verify (checkArgComparison "let") "let x = 0" +prop_checkArgComparison9 = verify (checkArgComparison "alias") "alias x =0" -- This mirrors checkSecondArgIsComparison but for arguments to local/readonly/declare/export checkArgComparison cmd = CommandCheck (Exactly cmd) wordsWithEqual where From fa15c0a454be93603418a56ed0d625e5c76a83fc Mon Sep 17 00:00:00 2001 From: Patrick Xia Date: Thu, 5 May 2022 16:09:02 -0700 Subject: [PATCH 227/459] add SC2316: error on multiple declarations like 'readonly local' --- src/ShellCheck/Checks/Commands.hs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 72d8c09..bad4cf2 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -100,6 +100,7 @@ commandChecks = [ ] ++ map checkArgComparison ("alias" : declaringCommands) ++ map checkMaskedReturns declaringCommands + ++ map checkMultipleDeclaring declaringCommands optionalChecks = map fst optionalCommandChecks @@ -941,6 +942,21 @@ checkLocalScope = CommandCheck (Exactly "local") $ \t -> unless (any isFunctionLike path) $ err (getId $ getCommandTokenOrThis t) 2168 "'local' is only valid in functions." +prop_checkMultipleDeclaring1 = verify (checkMultipleDeclaring "local") "q() { local readonly var=1; }" +prop_checkMultipleDeclaring2 = verifyNot (checkMultipleDeclaring "local") "q() { local var=1; }" +prop_checkMultipleDeclaring3 = verify (checkMultipleDeclaring "readonly") "readonly local foo=5" +prop_checkMultipleDeclaring4 = verify (checkMultipleDeclaring "export") "export readonly foo=5" +prop_checkMultipleDeclaring5 = verifyNot (checkMultipleDeclaring "local") "f() { local -r foo=5; }" +prop_checkMultipleDeclaring6 = verifyNot (checkMultipleDeclaring "declare") "declare -rx foo=5" +checkMultipleDeclaring cmd = CommandCheck (Exactly cmd) (mapM_ check . arguments) + where + check t = sequence_ $ do + lit <- getLiteralString t + guard $ lit `elem` declaringCommands + return $ err (getId $ getCommandTokenOrThis t) 2316 $ + "This applies " ++ cmd ++ " to the variable named " ++ lit ++ + ", which is probably not what you want. Use a separate command or the appropriate `declare` options instead." + prop_checkDeprecatedTempfile1 = verify checkDeprecatedTempfile "var=$(tempfile)" prop_checkDeprecatedTempfile2 = verifyNot checkDeprecatedTempfile "tempfile=$(mktemp)" checkDeprecatedTempfile = CommandCheck (Basename "tempfile") $ From 282155268829ffe450a21eb728ba0417c9c2578d Mon Sep 17 00:00:00 2001 From: Rune Juhl Jacobsen Date: Tue, 14 Dec 2021 16:00:47 +0100 Subject: [PATCH 228/459] Fix bug in 2126 when using after/before flags with grep Using `--after-context`/`-A` or `--before-context`/`-B` would give a warning recommending the user to use `grep -c`, even though that would give a different result than using `grep | wc -l`: ```fundamental $ echo -e "1\n2\n3" | grep -cA 3 1 1 $ echo -e "1\n2\n3" | grep -A 3 1 | wc -l 3 ``` --- src/ShellCheck/Analytics.hs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index d48104e..e6e47ea 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -545,6 +545,10 @@ prop_checkPipePitfalls15 = verifyNot checkPipePitfalls "foo | grep bar | wc -cmw prop_checkPipePitfalls16 = verifyNot checkPipePitfalls "foo | grep -r bar | wc -l" prop_checkPipePitfalls17 = verifyNot checkPipePitfalls "foo | grep -l bar | wc -l" prop_checkPipePitfalls18 = verifyNot checkPipePitfalls "foo | grep -L bar | wc -l" +prop_checkPipePitfalls19 = verifyNot checkPipePitfalls "foo | grep -A2 bar | wc -l" +prop_checkPipePitfalls20 = verifyNot checkPipePitfalls "foo | grep -B999 bar | wc -l" +prop_checkPipePitfalls21 = verifyNot checkPipePitfalls "foo | grep --after-context 999 bar | wc -l" +prop_checkPipePitfalls22 = verifyNot checkPipePitfalls "foo | grep -B 1 --after-context 999 bar | wc -l" checkPipePitfalls _ (T_Pipeline id _ commands) = do for ["find", "xargs"] $ \(find:xargs:_) -> @@ -566,10 +570,10 @@ checkPipePitfalls _ (T_Pipeline id _ commands) = do let flagsGrep = maybe [] (map snd . getAllFlags) $ getCommand grep flagsWc = maybe [] (map snd . getAllFlags) $ getCommand wc in - unless (any (`elem` ["l", "files-with-matches", "L", "files-without-matches", "o", "only-matching", "r", "R", "recursive"]) flagsGrep + unless (any (`elem` ["l", "files-with-matches", "L", "files-without-matches", "o", "only-matching", "r", "R", "recursive", "A", "after-context", "B", "before-context"]) flagsGrep || any (`elem` ["m", "chars", "w", "words", "c", "bytes", "L", "max-line-length"]) flagsWc || null flagsWc) $ - style (getId grep) 2126 "Consider using grep -c instead of grep|wc -l." + style (getId grep) 2126 "Consider using 'grep -c' instead of 'grep|wc -l'." didLs <- fmap or . sequence $ [ for' ["ls", "grep"] $ From fd595d1058a7991a3e925922be2da96c3c023faa Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Fri, 6 May 2022 10:06:12 -0700 Subject: [PATCH 229/459] Only trigger SC2316 on unquoted words. --- src/ShellCheck/Checks/Commands.hs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index bad4cf2..e65dc68 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -948,10 +948,11 @@ prop_checkMultipleDeclaring3 = verify (checkMultipleDeclaring "readonly") "reado prop_checkMultipleDeclaring4 = verify (checkMultipleDeclaring "export") "export readonly foo=5" prop_checkMultipleDeclaring5 = verifyNot (checkMultipleDeclaring "local") "f() { local -r foo=5; }" prop_checkMultipleDeclaring6 = verifyNot (checkMultipleDeclaring "declare") "declare -rx foo=5" +prop_checkMultipleDeclaring7 = verifyNot (checkMultipleDeclaring "readonly") "readonly 'local' foo=5" checkMultipleDeclaring cmd = CommandCheck (Exactly cmd) (mapM_ check . arguments) where check t = sequence_ $ do - lit <- getLiteralString t + lit <- getUnquotedLiteral t guard $ lit `elem` declaringCommands return $ err (getId $ getCommandTokenOrThis t) 2316 $ "This applies " ++ cmd ++ " to the variable named " ++ lit ++ From 399c04cc17931c5221672257666f39eda6aefc14 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Fri, 6 May 2022 10:11:52 -0700 Subject: [PATCH 230/459] Mention SC2316 in changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f65bbfa..fb733ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ## Git ### Added +- SC2316: Warn about 'local readonly foo' and similar (thanks, patrickxia!) ### Fixed From 9aa4c22aa6fb54d511148bdc8b135e353529ffcc Mon Sep 17 00:00:00 2001 From: Frazer Smith Date: Mon, 16 May 2022 06:56:46 +0000 Subject: [PATCH 231/459] ci: update github actions --- .github/workflows/build.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5595219..a435cf4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,7 +15,7 @@ jobs: sudo apt-get install cabal-install - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 @@ -37,7 +37,7 @@ jobs: mv dist-newstyle/sdist/*.tar.gz source/source.tar.gz - name: Upload artifact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: source path: source/ @@ -51,10 +51,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Download artifacts - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 - name: Build source run: | @@ -63,7 +63,7 @@ jobs: ( cd bin && ../build/run_builder ../source/source.tar.gz ../build/${{matrix.build}} ) - name: Upload artifact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: bin path: bin/ @@ -74,10 +74,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Download artifacts - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 - name: Work around GitHub permissions bug run: chmod +x bin/*/shellcheck* @@ -92,7 +92,7 @@ jobs: rm -rf */ README* LICENSE* - name: Upload artifact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: deploy path: deploy/ @@ -104,10 +104,10 @@ jobs: environment: Deploy steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Download artifacts - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 - name: Upload to GitHub env: From 7ceb1f15193f43109a547ccc3d8ba74faaa72ade Mon Sep 17 00:00:00 2001 From: ygeyzel Date: Sun, 17 Jul 2022 21:46:42 +0300 Subject: [PATCH 232/459] SC2183 grammer fix: 'variable' instead of 'variables' if only one variable --- src/ShellCheck/Checks/Commands.hs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index e65dc68..d8635ea 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -675,6 +675,7 @@ checkPrintfVar = CommandCheck (Exactly "printf") (f . arguments) where let formats = getPrintfFormats string let formatCount = length formats let argCount = length more + let pluraliseIfMany word n = if n > 1 then word ++ "s" else word return $ if | argCount == 0 && formatCount == 0 -> @@ -690,7 +691,8 @@ checkPrintfVar = CommandCheck (Exactly "printf") (f . arguments) where return () -- Great: a suitable number of arguments | otherwise -> warn (getId format) 2183 $ - "This format string has " ++ show formatCount ++ " variables, but is passed " ++ show argCount ++ " arguments." + "This format string has " ++ show formatCount ++ " " ++ (pluraliseIfMany "variable" formatCount) ++ + ", but is passed " ++ show argCount ++ " arguments." unless ('%' `elem` concat (oversimplify format) || isLiteral format) $ info (getId format) 2059 From 363c0633e0525e6e7d1714cac83e420875345b85 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Fri, 11 Feb 2022 17:17:04 -0800 Subject: [PATCH 233/459] When reparsing array indices, do it recursively --- src/ShellCheck/Analytics.hs | 1 + src/ShellCheck/Parser.hs | 9 +++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index e6e47ea..7da5786 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2394,6 +2394,7 @@ prop_checkUnused47= verifyNotTree checkUnusedAssignments "a=1; alias hello='echo prop_checkUnused48= verifyNotTree checkUnusedAssignments "_a=1" prop_checkUnused49= verifyNotTree checkUnusedAssignments "declare -A array; key=a; [[ -v array[$key] ]]" prop_checkUnused50= verifyNotTree checkUnusedAssignments "foofunc() { :; }; typeset -fx foofunc" +prop_checkUnused51= verifyTree checkUnusedAssignments "x[y[z=1]]=1; echo ${x[@]}" checkUnusedAssignments params t = execWriter (mapM_ warnFor unused) where diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 8fbdb5a..833918c 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -3463,9 +3463,9 @@ notesForContext list = zipWith ($) [first, second] $ filter isName list -- Go over all T_UnparsedIndex and reparse them as either arithmetic or text -- depending on declare -A statements. -reparseIndices root = - analyze blank blank f root +reparseIndices root = process root where + process = analyze blank blank f associative = getAssociativeArrays root isAssociative s = s `elem` associative f (T_Assignment id mode name indices value) = do @@ -3490,8 +3490,9 @@ reparseIndices root = fixAssignmentIndex name word = case word of - T_UnparsedIndex id pos src -> - parsed name pos src + T_UnparsedIndex id pos src -> do + idx <- parsed name pos src + process idx -- Recursively parse for cases like x[y[z=1]]=1 _ -> return word parsed name pos src = From a4042f752399b0aff032692331e9f561c2cb836c Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 18 Jul 2022 22:12:31 -0700 Subject: [PATCH 234/459] Parse &&/|| as left-associative --- src/ShellCheck/Analytics.hs | 2 +- src/ShellCheck/Parser.hs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 7da5786..f5ff4df 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -867,7 +867,7 @@ prop_checkShorthandIf5 = verifyNot checkShorthandIf "foo && rm || printf b" prop_checkShorthandIf6 = verifyNot checkShorthandIf "if foo && bar || baz; then true; fi" prop_checkShorthandIf7 = verifyNot checkShorthandIf "while foo && bar || baz; do true; done" prop_checkShorthandIf8 = verify checkShorthandIf "if true; then foo && bar || baz; fi" -checkShorthandIf params x@(T_AndIf id _ (T_OrIf _ _ (T_Pipeline _ _ t))) +checkShorthandIf params x@(T_OrIf _ (T_AndIf id _ _) (T_Pipeline _ _ t)) | not (isOk t || inCondition) = info id 2015 "Note that A && B || C is not if-then-else. C may run when A is true." where diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 833918c..3958406 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -2286,7 +2286,7 @@ readAndOr = do parseProblemAt apos ErrorC 1123 "ShellCheck directives are only valid in front of complete compound commands, like 'if', not e.g. individual 'elif' branches." andOr <- withAnnotations annotations $ - chainr1 readPipeline $ do + chainl1 readPipeline $ do op <- g_AND_IF <|> g_OR_IF readLineBreak return $ case op of T_AND_IF id -> T_AndIf id From c3bce51de38fa57b006b932661bdb6237e0c5db5 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 19 Jul 2022 17:45:54 -0700 Subject: [PATCH 235/459] Allow text to build on Fedora by installing dependencies --- test/distrotest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/distrotest b/test/distrotest index 50a5a17..464768c 100755 --- a/test/distrotest +++ b/test/distrotest @@ -63,7 +63,7 @@ debian:testing apt-get update && apt-get install -y cabal-install ubuntu:latest apt-get update && apt-get install -y cabal-install haskell:latest true opensuse/leap:latest zypper install -y cabal-install ghc -fedora:latest dnf install -y cabal-install ghc-template-haskell-devel findutils +fedora:latest dnf install -y cabal-install ghc-template-haskell-devel findutils libstdc++-static gcc-c++ archlinux:latest pacman -S -y --noconfirm cabal-install ghc-static base-devel # Ubuntu LTS From cc04b4011967f873e14c0e6ad506ca37bed1f97f Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 19 Jul 2022 18:22:11 -0700 Subject: [PATCH 236/459] Freeze macOS dependency by sha256 --- build/darwin.x86_64/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/build/darwin.x86_64/Dockerfile b/build/darwin.x86_64/Dockerfile index ecd1cad..9e33a82 100644 --- a/build/darwin.x86_64/Dockerfile +++ b/build/darwin.x86_64/Dockerfile @@ -1,5 +1,4 @@ -# DIGEST:sha256:fa32af4677e2860a1c5950bc8c360f309e2a87e2ddfed27b642fddf7a6093b76 -FROM liushuyu/osxcross:latest +FROM liushuyu/osxcross@sha256:fa32af4677e2860a1c5950bc8c360f309e2a87e2ddfed27b642fddf7a6093b76 ENV TARGET x86_64-apple-darwin18 ENV TARGETNAME darwin.x86_64 From 7946bf5657905bba74be360bf16b287008ed1bdb Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 19 Jul 2022 21:40:03 -0700 Subject: [PATCH 237/459] Upgrade cURL for Windows build image --- build/windows.x86_64/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/windows.x86_64/Dockerfile b/build/windows.x86_64/Dockerfile index 11e67e8..1e5c5d9 100644 --- a/build/windows.x86_64/Dockerfile +++ b/build/windows.x86_64/Dockerfile @@ -12,7 +12,7 @@ WORKDIR /haskell RUN curl -L "https://downloads.haskell.org/~ghc/8.10.4/ghc-8.10.4-x86_64-unknown-mingw32.tar.xz" | tar xJ --strip-components=1 WORKDIR /haskell/bin RUN curl -L "https://downloads.haskell.org/~cabal/cabal-install-3.2.0.0/cabal-install-3.2.0.0-x86_64-unknown-mingw32.zip" | busybox unzip - -RUN curl -L "https://curl.se/windows/dl-7.75.0/curl-7.75.0-win64-mingw.zip" | busybox unzip - && mv curl-7.75.0-win64-mingw/bin/* . +RUN curl -L "https://curl.se/windows/dl-7.84.0/curl-7.84.0-win64-mingw.zip" | busybox unzip - && mv curl-7.84.0-win64-mingw/bin/* . ENV WINEPATH /haskell/bin # It's unknown whether Cabal on Windows suffers from the same issue From f77a545282f02010a0c52b66bd6ee5e860c2b314 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 19 Jul 2022 14:23:27 -0700 Subject: [PATCH 238/459] Control Flow Graph / Data Flow Analysis support --- CHANGELOG.md | 4 + ShellCheck.cabal | 8 + src/ShellCheck/ASTLib.hs | 116 ++- src/ShellCheck/Analytics.hs | 2 + src/ShellCheck/Analyzer.hs | 5 +- src/ShellCheck/AnalyzerLib.hs | 142 +-- src/ShellCheck/CFG.hs | 1147 +++++++++++++++++++++++++ src/ShellCheck/CFGAnalysis.hs | 1113 ++++++++++++++++++++++++ src/ShellCheck/Checks/Commands.hs | 1 + src/ShellCheck/Checks/ControlFlow.hs | 101 +++ src/ShellCheck/Checks/ShellSupport.hs | 1 + src/ShellCheck/Data.hs | 29 +- src/ShellCheck/Debug.hs | 313 +++++++ src/ShellCheck/Fixer.hs | 3 +- src/ShellCheck/Parser.hs | 5 +- src/ShellCheck/Prelude.hs | 48 ++ test/shellcheck.hs | 6 + 17 files changed, 2909 insertions(+), 135 deletions(-) create mode 100644 src/ShellCheck/CFG.hs create mode 100644 src/ShellCheck/CFGAnalysis.hs create mode 100644 src/ShellCheck/Checks/ControlFlow.hs create mode 100644 src/ShellCheck/Debug.hs create mode 100644 src/ShellCheck/Prelude.hs diff --git a/CHANGELOG.md b/CHANGELOG.md index fb733ce..4763ddd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ ### Fixed ### Changed +- ShellCheck now has a Data Flow Analysis engine to make smarter decisions + based on control flow rather than just syntax. Existing checks will + gradually start using it, which may cause them to trigger differently + (but more accurately). ## v0.8.0 - 2021-11-06 diff --git a/ShellCheck.cabal b/ShellCheck.cabal index 1167c82..b22b5c8 100644 --- a/ShellCheck.cabal +++ b/ShellCheck.cabal @@ -53,6 +53,7 @@ library deepseq >= 1.4.0.0, Diff >= 0.2.0, directory >= 1.2.3.0, + fgl, mtl >= 2.2.1, filepath, parsec, @@ -66,11 +67,15 @@ library ShellCheck.Analytics ShellCheck.Analyzer ShellCheck.AnalyzerLib + ShellCheck.CFG + ShellCheck.CFGAnalysis ShellCheck.Checker ShellCheck.Checks.Commands + ShellCheck.Checks.ControlFlow ShellCheck.Checks.Custom ShellCheck.Checks.ShellSupport ShellCheck.Data + ShellCheck.Debug ShellCheck.Fixer ShellCheck.Formatter.Format ShellCheck.Formatter.CheckStyle @@ -82,6 +87,7 @@ library ShellCheck.Formatter.Quiet ShellCheck.Interface ShellCheck.Parser + ShellCheck.Prelude ShellCheck.Regex other-modules: Paths_ShellCheck @@ -100,6 +106,7 @@ executable shellcheck deepseq >= 1.4.0.0, Diff >= 0.2.0, directory >= 1.2.3.0, + fgl, mtl >= 2.2.1, filepath, parsec >= 3.0, @@ -120,6 +127,7 @@ test-suite test-shellcheck deepseq >= 1.4.0.0, Diff >= 0.2.0, directory >= 1.2.3.0, + fgl, mtl >= 2.2.1, filepath, parsec, diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 7c88432..7cc5af2 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -21,6 +21,7 @@ module ShellCheck.ASTLib where import ShellCheck.AST +import ShellCheck.Prelude import ShellCheck.Regex import Control.Monad.Writer @@ -138,7 +139,7 @@ getFlagsUntil stopCondition (T_SimpleCommand _ _ (_:args)) = flag (x, '-':'-':arg) = [ (x, takeWhile (/= '=') arg) ] flag (x, '-':args) = map (\v -> (x, [v])) args flag (x, _) = [ (x, "") ] -getFlagsUntil _ _ = error "Internal shellcheck error, please report! (getFlags on non-command)" +getFlagsUntil _ _ = error $ pleaseReport "getFlags on non-command" -- Get all flags in a GNU way, up until -- getAllFlags :: Token -> [(Token, String)] @@ -785,5 +786,118 @@ executableFromShebang = shellFor basename s = reverse . takeWhile (/= '/') . reverse $ s skipFlags = dropWhile ("-" `isPrefixOf`) + +-- Determining if a name is a variable +isVariableStartChar x = x == '_' || isAsciiLower x || isAsciiUpper x +isVariableChar x = isVariableStartChar x || isDigit x +isSpecialVariableChar = (`elem` "*@#?-$!") +variableNameRegex = mkRegex "[_a-zA-Z][_a-zA-Z0-9]*" + +prop_isVariableName1 = isVariableName "_fo123" +prop_isVariableName2 = not $ isVariableName "4" +prop_isVariableName3 = not $ isVariableName "test: " +isVariableName (x:r) = isVariableStartChar x && all isVariableChar r +isVariableName _ = False + + +-- Get the variable name from an expansion like ${var:-foo} +prop_getBracedReference1 = getBracedReference "foo" == "foo" +prop_getBracedReference2 = getBracedReference "#foo" == "foo" +prop_getBracedReference3 = getBracedReference "#" == "#" +prop_getBracedReference4 = getBracedReference "##" == "#" +prop_getBracedReference5 = getBracedReference "#!" == "!" +prop_getBracedReference6 = getBracedReference "!#" == "#" +prop_getBracedReference7 = getBracedReference "!foo#?" == "foo" +prop_getBracedReference8 = getBracedReference "foo-bar" == "foo" +prop_getBracedReference9 = getBracedReference "foo:-bar" == "foo" +prop_getBracedReference10= getBracedReference "foo: -1" == "foo" +prop_getBracedReference11= getBracedReference "!os*" == "" +prop_getBracedReference11b= getBracedReference "!os@" == "" +prop_getBracedReference12= getBracedReference "!os?bar**" == "" +prop_getBracedReference13= getBracedReference "foo[bar]" == "foo" +getBracedReference s = fromMaybe s $ + nameExpansion s `mplus` takeName noPrefix `mplus` getSpecial noPrefix `mplus` getSpecial s + where + noPrefix = dropPrefix s + dropPrefix (c:rest) | c `elem` "!#" = rest + dropPrefix cs = cs + takeName s = do + let name = takeWhile isVariableChar s + guard . not $ null name + return name + getSpecial (c:_) | isSpecialVariableChar c = return [c] + getSpecial _ = fail "empty or not special" + + nameExpansion ('!':next:rest) = do -- e.g. ${!foo*bar*} + guard $ isVariableChar next -- e.g. ${!@} + first <- find (not . isVariableChar) rest + guard $ first `elem` "*?@" + return "" + nameExpansion _ = Nothing + +-- Get the variable modifier like /a/b in ${var/a/b} +prop_getBracedModifier1 = getBracedModifier "foo:bar:baz" == ":bar:baz" +prop_getBracedModifier2 = getBracedModifier "!var:-foo" == ":-foo" +prop_getBracedModifier3 = getBracedModifier "foo[bar]" == "[bar]" +prop_getBracedModifier4 = getBracedModifier "foo[@]@Q" == "[@]@Q" +prop_getBracedModifier5 = getBracedModifier "@@Q" == "@Q" +getBracedModifier s = headOrDefault "" $ do + let var = getBracedReference s + a <- dropModifier s + dropPrefix var a + where + dropPrefix [] t = return t + dropPrefix (a:b) (c:d) | a == c = dropPrefix b d + dropPrefix _ _ = [] + + dropModifier (c:rest) | c `elem` "#!" = [rest, c:rest] + dropModifier x = [x] + +-- Get the variables from indices like ["x", "y"] in ${var[x+y+1]} +prop_getIndexReferences1 = getIndexReferences "var[x+y+1]" == ["x", "y"] +getIndexReferences s = fromMaybe [] $ do + match <- matchRegex re s + index <- match !!! 0 + return $ matchAllStrings variableNameRegex index + where + re = mkRegex "(\\[.*\\])" + +prop_getOffsetReferences1 = getOffsetReferences ":bar" == ["bar"] +prop_getOffsetReferences2 = getOffsetReferences ":bar:baz" == ["bar", "baz"] +prop_getOffsetReferences3 = getOffsetReferences "[foo]:bar" == ["bar"] +prop_getOffsetReferences4 = getOffsetReferences "[foo]:bar:baz" == ["bar", "baz"] +getOffsetReferences mods = fromMaybe [] $ do +-- if mods start with [, then drop until ] + match <- matchRegex re mods + offsets <- match !!! 1 + return $ matchAllStrings variableNameRegex offsets + where + re = mkRegex "^(\\[.+\\])? *:([^-=?+].*)" + + +-- Returns whether a token is a parameter expansion without any modifiers. +-- True for $var ${var} $1 $# +-- False for ${#var} ${var[x]} ${var:-0} +isUnmodifiedParameterExpansion t = + case t of + T_DollarBraced _ False _ -> True + T_DollarBraced _ _ list -> + let str = concat $ oversimplify list + in getBracedReference str == str + _ -> False + +--- A list of the element and all its parents up to the root node. +getPath tree t = t : + case Map.lookup (getId t) tree of + Nothing -> [] + Just parent -> getPath tree parent + +isClosingFileOp op = + case op of + T_IoDuplicate _ (T_GREATAND _) "-" -> True + T_IoDuplicate _ (T_LESSAND _) "-" -> True + _ -> False + + return [] runTests = $quickCheckAll diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index f5ff4df..bf5d179 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -24,8 +24,10 @@ module ShellCheck.Analytics (runAnalytics, optionalChecks, ShellCheck.Analytics. import ShellCheck.AST import ShellCheck.ASTLib import ShellCheck.AnalyzerLib hiding (producesComments) +import qualified ShellCheck.CFGAnalysis as CF import ShellCheck.Data import ShellCheck.Parser +import ShellCheck.Prelude import ShellCheck.Interface import ShellCheck.Regex diff --git a/src/ShellCheck/Analyzer.hs b/src/ShellCheck/Analyzer.hs index eb231c2..ff2e457 100644 --- a/src/ShellCheck/Analyzer.hs +++ b/src/ShellCheck/Analyzer.hs @@ -25,6 +25,7 @@ import ShellCheck.Interface import Data.List import Data.Monoid import qualified ShellCheck.Checks.Commands +import qualified ShellCheck.Checks.ControlFlow import qualified ShellCheck.Checks.Custom import qualified ShellCheck.Checks.ShellSupport @@ -42,11 +43,13 @@ analyzeScript spec = newAnalysisResult { checkers spec params = mconcat $ map ($ params) [ ShellCheck.Checks.Commands.checker spec, + ShellCheck.Checks.ControlFlow.checker spec, ShellCheck.Checks.Custom.checker, ShellCheck.Checks.ShellSupport.checker ] optionalChecks = mconcat $ [ ShellCheck.Analytics.optionalChecks, - ShellCheck.Checks.Commands.optionalChecks + ShellCheck.Checks.Commands.optionalChecks, + ShellCheck.Checks.ControlFlow.optionalChecks ] diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 67c35b4..e998f2c 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -23,9 +23,11 @@ module ShellCheck.AnalyzerLib where import ShellCheck.AST import ShellCheck.ASTLib +import qualified ShellCheck.CFGAnalysis as CF import ShellCheck.Data import ShellCheck.Interface import ShellCheck.Parser +import ShellCheck.Prelude import ShellCheck.Regex import Control.Arrow (first) @@ -96,7 +98,9 @@ data Parameters = Parameters { -- The root node of the AST rootNode :: Token, -- map from token id to start and end position - tokenPositions :: Map.Map Id (Position, Position) + tokenPositions :: Map.Map Id (Position, Position), + -- Result from Control Flow Graph analysis (including data flow analysis) + cfgAnalysis :: CF.CFGAnalysis } deriving (Show) -- TODO: Cache results of common AST ops here @@ -189,8 +193,9 @@ makeCommentWithFix severity id code str fix = } in force withFix -makeParameters spec = - let params = Parameters { +makeParameters spec = params + where + params = Parameters { rootNode = root, shellType = fromMaybe (determineShell (asFallbackShell spec) root) $ asShellType spec, hasSetE = containsSetE root, @@ -215,9 +220,14 @@ makeParameters spec = shellTypeSpecified = isJust (asShellType spec) || isJust (asFallbackShell spec), parentMap = getParentTree root, variableFlow = getVariableFlow params root, - tokenPositions = asTokenPositions spec - } in params - where root = asScript spec + tokenPositions = asTokenPositions spec, + cfgAnalysis = CF.analyzeControlFlow cfParams root + } + cfParams = CF.CFGParameters { + CF.cfLastpipe = hasLastpipe params, + CF.cfPipefail = hasPipefail params + } + root = asScript spec -- Does this script mention 'set -e' anywhere? @@ -408,12 +418,6 @@ usedAsCommandName tree token = go (getId token) (tail $ getPath tree token) getId word == currentId || getId (getCommandTokenOrThis t) == currentId go _ _ = False --- A list of the element and all its parents up to the root node. -getPath tree t = t : - case Map.lookup (getId t) tree of - Nothing -> [] - Just parent -> getPath tree parent - -- Version of the above taking the map from the current context -- Todo: give this the name "getPath" getPathM t = do @@ -559,12 +563,6 @@ getModifiedVariables t = return (place, t, str, DataString SourceChecked) _ -> Nothing -isClosingFileOp op = - case op of - T_IoDuplicate _ (T_GREATAND _) "-" -> True - T_IoDuplicate _ (T_LESSAND _) "-" -> True - _ -> False - -- Consider 'export/declare -x' a reference, since it makes the var available getReferencedVariableCommand base@(T_SimpleCommand _ _ (T_NormalWord _ (T_Literal _ x:_):rest)) = @@ -746,13 +744,6 @@ getModifiedVariableCommand base@(T_SimpleCommand id cmdPrefix (T_NormalWord _ (T getModifiedVariableCommand _ = [] -getIndexReferences s = fromMaybe [] $ do - match <- matchRegex re s - index <- match !!! 0 - return $ matchAllStrings variableNameRegex index - where - re = mkRegex "(\\[.*\\])" - -- Given a NormalWord like foo or foo[$bar], get foo. -- Primarily used to get references for [[ -v foo[bar] ]] getVariableForTestDashV :: Token -> Maybe String @@ -767,18 +758,6 @@ getVariableForTestDashV t = do -- in a non-constant expression (while filtering out foo$x[$y]) toStr _ = return "\0" -prop_getOffsetReferences1 = getOffsetReferences ":bar" == ["bar"] -prop_getOffsetReferences2 = getOffsetReferences ":bar:baz" == ["bar", "baz"] -prop_getOffsetReferences3 = getOffsetReferences "[foo]:bar" == ["bar"] -prop_getOffsetReferences4 = getOffsetReferences "[foo]:bar:baz" == ["bar", "baz"] -getOffsetReferences mods = fromMaybe [] $ do --- if mods start with [, then drop until ] - match <- matchRegex re mods - offsets <- match !!! 1 - return $ matchAllStrings variableNameRegex offsets - where - re = mkRegex "^(\\[.+\\])? *:([^-=?+].*)" - getReferencedVariables parents t = case t of T_DollarBraced id _ l -> let str = concat $ oversimplify l in @@ -857,17 +836,6 @@ isConfusedGlobRegex ('*':_) = True isConfusedGlobRegex [x,'*'] | x `notElem` "\\." = True isConfusedGlobRegex _ = False -isVariableStartChar x = x == '_' || isAsciiLower x || isAsciiUpper x -isVariableChar x = isVariableStartChar x || isDigit x -isSpecialVariableChar = (`elem` "*@#?-$!") -variableNameRegex = mkRegex "[_a-zA-Z][_a-zA-Z0-9]*" - -prop_isVariableName1 = isVariableName "_fo123" -prop_isVariableName2 = not $ isVariableName "4" -prop_isVariableName3 = not $ isVariableName "test: " -isVariableName (x:r) = isVariableStartChar x && all isVariableChar r -isVariableName _ = False - getVariablesFromLiteralToken token = getVariablesFromLiteral (getLiteralStringDef " " token) @@ -880,73 +848,6 @@ getVariablesFromLiteral string = where variableRegex = mkRegex "\\$\\{?([A-Za-z0-9_]+)" --- Get the variable name from an expansion like ${var:-foo} -prop_getBracedReference1 = getBracedReference "foo" == "foo" -prop_getBracedReference2 = getBracedReference "#foo" == "foo" -prop_getBracedReference3 = getBracedReference "#" == "#" -prop_getBracedReference4 = getBracedReference "##" == "#" -prop_getBracedReference5 = getBracedReference "#!" == "!" -prop_getBracedReference6 = getBracedReference "!#" == "#" -prop_getBracedReference7 = getBracedReference "!foo#?" == "foo" -prop_getBracedReference8 = getBracedReference "foo-bar" == "foo" -prop_getBracedReference9 = getBracedReference "foo:-bar" == "foo" -prop_getBracedReference10= getBracedReference "foo: -1" == "foo" -prop_getBracedReference11= getBracedReference "!os*" == "" -prop_getBracedReference11b= getBracedReference "!os@" == "" -prop_getBracedReference12= getBracedReference "!os?bar**" == "" -prop_getBracedReference13= getBracedReference "foo[bar]" == "foo" -getBracedReference s = fromMaybe s $ - nameExpansion s `mplus` takeName noPrefix `mplus` getSpecial noPrefix `mplus` getSpecial s - where - noPrefix = dropPrefix s - dropPrefix (c:rest) | c `elem` "!#" = rest - dropPrefix cs = cs - takeName s = do - let name = takeWhile isVariableChar s - guard . not $ null name - return name - getSpecial (c:_) | isSpecialVariableChar c = return [c] - getSpecial _ = fail "empty or not special" - - nameExpansion ('!':next:rest) = do -- e.g. ${!foo*bar*} - guard $ isVariableChar next -- e.g. ${!@} - first <- find (not . isVariableChar) rest - guard $ first `elem` "*?@" - return "" - nameExpansion _ = Nothing - -prop_getBracedModifier1 = getBracedModifier "foo:bar:baz" == ":bar:baz" -prop_getBracedModifier2 = getBracedModifier "!var:-foo" == ":-foo" -prop_getBracedModifier3 = getBracedModifier "foo[bar]" == "[bar]" -prop_getBracedModifier4 = getBracedModifier "foo[@]@Q" == "[@]@Q" -prop_getBracedModifier5 = getBracedModifier "@@Q" == "@Q" -getBracedModifier s = headOrDefault "" $ do - let var = getBracedReference s - a <- dropModifier s - dropPrefix var a - where - dropPrefix [] t = return t - dropPrefix (a:b) (c:d) | a == c = dropPrefix b d - dropPrefix _ _ = [] - - dropModifier (c:rest) | c `elem` "#!" = [rest, c:rest] - dropModifier x = [x] - --- Useful generic functions. - --- Get element 0 or a default. Like `head` but safe. -headOrDefault _ (a:_) = a -headOrDefault def _ = def - --- Get the last element or a default. Like `last` but safe. -lastOrDefault def [] = def -lastOrDefault _ list = last list - ---- Get element n of a list, or Nothing. Like `!!` but safe. -(!!!) list i = - case drop i list of - [] -> Nothing - (r:_) -> Just r -- Run a command if the shell is in the given list whenShell l c = do @@ -999,17 +900,6 @@ isBashLike params = Dash -> False Sh -> False --- Returns whether a token is a parameter expansion without any modifiers. --- True for $var ${var} $1 $# --- False for ${#var} ${var[x]} ${var:-0} -isUnmodifiedParameterExpansion t = - case t of - T_DollarBraced _ False _ -> True - T_DollarBraced _ _ list -> - let str = concat $ oversimplify list - in getBracedReference str == str - _ -> False - isTrueAssignmentSource c = case c of DataString SourceChecked -> False diff --git a/src/ShellCheck/CFG.hs b/src/ShellCheck/CFG.hs new file mode 100644 index 0000000..101a0d7 --- /dev/null +++ b/src/ShellCheck/CFG.hs @@ -0,0 +1,1147 @@ +{- + Copyright 2022 Vidar Holen + + This file is part of ShellCheck. + https://www.shellcheck.net + + ShellCheck is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + ShellCheck is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +-} +{-# LANGUAGE TemplateHaskell #-} +{-# LANGUAGE DeriveAnyClass, DeriveGeneric #-} + +-- Constructs a Control Flow Graph from an AST +module ShellCheck.CFG ( + CFNode (..), + CFEdge (..), + CFEffect (..), + CFStringPart (..), + CFVariableProp (..), + CFGResult (..), + CFValue (..), + CFGraph, + CFGParameters (..), + IdTagged (..), + buildGraph + , ShellCheck.CFG.runTests -- STRIP + ) + where + +import GHC.Generics (Generic) +import ShellCheck.AST +import ShellCheck.ASTLib +import ShellCheck.Interface +import ShellCheck.Prelude +import ShellCheck.Regex +import Control.DeepSeq +import Control.Monad +import Control.Monad.Identity +import Data.List hiding (map) +import Data.Maybe +import qualified Data.Map as M +import qualified Data.Set as S +import Control.Monad.RWS.Lazy +import Data.Graph.Inductive.Graph +import Data.Graph.Inductive.Query.DFS +import Data.Graph.Inductive.PatriciaTree as G +import Debug.Trace -- STRIP + +import Test.QuickCheck.All (forAllProperties) +import Test.QuickCheck.Test (quickCheckWithResult, stdArgs, maxSuccess) + + +-- Our basic Graph type +type CFGraph = G.Gr CFNode CFEdge + +-- Node labels in a Control Flow Graph +data CFNode = + -- A no-op node for structural purposes + CFStructuralNode + -- A no-op for graph inspection purposes + | CFEntryPoint String + -- Drop current prefix assignments + | CFDropPrefixAssignments + -- A node with a certain effect on program state + | CFApplyEffects [IdTagged CFEffect] + -- The execution of a command or function by literal string if possible + | CFExecuteCommand (Maybe String) + -- Execute a subshell. These are represented by disjoint graphs just like + -- functions, but they don't require any form of name resolution + | CFExecuteSubshell String Node Node + -- Assignment of $? + | CFSetExitCode Id + -- The virtual 'exit' at the natural end of a subshell + | CFImpliedExit + -- An exit statement resolvable at CFG build time + | CFResolvedExit + -- An exit statement only resolvable at DFA time + | CFUnresolvedExit + -- An unreachable node, serving as the unconnected end point of a range + | CFUnreachable + -- Assignment of $! + | CFSetBackgroundPid Id + deriving (Eq, Ord, Show, Generic, NFData) + +-- Edge labels in a Control Flow Graph +data CFEdge = + CFEErrExit + -- Regular control flow edge + | CFEFlow + -- An edge that a human might think exists (e.g. from a backgrounded process to its parent) + | CFEFalseFlow + -- An edge followed on exit + | CFEExit + deriving (Eq, Ord, Show, Generic, NFData) + +-- Actions we track +data CFEffect = + CFModifyProps String [CFVariableProp] + | CFReadVariable String + | CFWriteVariable String CFValue + | CFWriteGlobal String CFValue + | CFWriteLocal String CFValue + | CFWritePrefix String CFValue + | CFDefineFunction String Id Node Node + | CFUndefine String + | CFUndefineVariable String + | CFUndefineFunction String + | CFUndefineNameref String + -- Usage implies that this is an array (e.g. it's expanded with index) + | CFHintArray String + -- Operation implies that the variable will be defined (e.g. [ -z "$var" ]) + | CFHintDefined String + deriving (Eq, Ord, Show, Generic, NFData) + +data IdTagged a = IdTagged Id a + deriving (Eq, Ord, Show, Generic, NFData) + +-- Where a variable's value comes from +data CFValue = + -- The special 'uninitialized' value + CFValueUninitialized + -- An arbitrary array value + | CFValueArray + -- An arbitrary string value + | CFValueString + -- An arbitrary integer + | CFValueInteger + -- Token 'Id' concatenates and assigns the given parts + | CFValueComputed Id [CFStringPart] + deriving (Eq, Ord, Show, Generic, NFData) + +-- Simplified computed strings +data CFStringPart = + -- A known literal string value, like 'foo' + CFStringLiteral String + -- The contents of a variable, like $foo + | CFStringVariable String + -- An value that is unknown but an integer + | CFStringInteger + -- An unknown string value, for things we can't handle + | CFStringUnknown + deriving (Eq, Ord, Show, Generic, NFData) + +-- The properties of a variable +data CFVariableProp = CFVPExport | CFVPArray + deriving (Eq, Ord, Show, Generic, NFData) + +-- Options when generating CFG +data CFGParameters = CFGParameters { + -- Whether the last element in a pipeline runs in the current shell + cfLastpipe :: Bool, + -- Whether all elements in a pipeline count towards the exit status + cfPipefail :: Bool +} + +data CFGResult = CFGResult { + -- The graph itself + cfGraph :: CFGraph, + -- Map from Id to start/end node + cfIdToNode :: M.Map Id (Node, Node) +} + deriving (Show) + +buildGraph :: CFGParameters -> Token -> CFGResult +buildGraph params root = + let + (nextNode, base) = execRWS (buildRoot root) (newCFContext params) 0 + (nodes, edges, mapping) = +-- renumberTopologically $ + removeUnnecessaryStructuralNodes + base + in + CFGResult { + cfGraph = mkGraph nodes edges, + cfIdToNode = M.fromList mapping + } + +remapGraph remap (nodes, edges, mapping) = + ( + map (remapNode remap) nodes, + map (remapEdge remap) edges, + map (\(id, (a,b)) -> (id, (remapHelper remap a, remapHelper remap b))) mapping + ) + +prop_testRenumbering = + let + s = CFStructuralNode + before = ( + [(1,s), (3,s), (4, s), (8,s)], + [(1,3,CFEFlow), (3,4, CFEFlow), (4,8,CFEFlow)], + [(Id 0, (3,4))] + ) + after = ( + [(0,s), (1,s), (2,s), (3,s)], + [(0,1,CFEFlow), (1,2, CFEFlow), (2,3,CFEFlow)], + [(Id 0, (1,2))] + ) + in after == renumberGraph before + +-- Renumber the graph for prettiness, so there are no gaps in node numbers +renumberGraph g@(nodes, edges, mapping) = + let renumbering = M.fromList (flip zip [0..] $ sort $ map fst nodes) + in remapGraph renumbering g + +prop_testRenumberTopologically = + let + s = CFStructuralNode + before = ( + [(4,s), (2,s), (3, s)], + [(4,2,CFEFlow), (2,3, CFEFlow)], + [(Id 0, (4,2))] + ) + after = ( + [(0,s), (1,s), (2,s)], + [(0,1,CFEFlow), (1,2, CFEFlow)], + [(Id 0, (0,1))] + ) + in after == renumberTopologically before + +-- Renumber the graph in topological order +renumberTopologically g@(nodes, edges, mapping) = + let renumbering = M.fromList (flip zip [0..] $ topsort (mkGraph nodes edges :: CFGraph)) + in remapGraph renumbering g + +prop_testRemoveStructural = + let + s = CFStructuralNode + before = ( + [(1,s), (2,s), (3, s), (4,s)], + [(1,2,CFEFlow), (2,3, CFEFlow), (3,4,CFEFlow)], + [(Id 0, (2,3))] + ) + after = ( + [(1,s), (2,s), (4,s)], + [(1,2,CFEFlow), (2,4,CFEFlow)], + [(Id 0, (2,2))] + ) + in after == removeUnnecessaryStructuralNodes before + +-- Collapse structural nodes that just form long chains like x->x->x. +-- This way we can generate them with abandon, without making DFA slower. +-- +-- Note in particular that we can't remove a structural node x in +-- foo -> x -> bar , because then the pre/post-condition for tokens +-- previously pointing to x would be wrong. +removeUnnecessaryStructuralNodes (nodes, edges, mapping) = + remapGraph recursiveRemapping + ( + filter (\(n, _) -> n `M.notMember` recursiveRemapping) nodes, + filter (`S.notMember` edgesToCollapse) edges, + mapping + ) + where + regularEdges = filter isRegularEdge edges + inDegree = counter $ map (\(from,to,_) -> from) regularEdges + outDegree = counter $ map (\(from,to,_) -> to) regularEdges + structuralNodes = S.fromList $ map fst $ filter isStructural nodes + candidateNodes = S.filter isLinear structuralNodes + edgesToCollapse = S.fromList $ filter filterEdges regularEdges + + remapping :: M.Map Node Node + remapping = foldl' (\m (new, old) -> M.insert old new m) M.empty $ map orderEdge $ S.toList edgesToCollapse + recursiveRemapping = M.fromList $ map (\c -> (c, recursiveLookup remapping c)) $ M.keys remapping + + filterEdges (a,b,_) = + a `S.member` candidateNodes && b `S.member` candidateNodes + + orderEdge (a,b,_) = if a < b then (a,b) else (b,a) + counter = foldl' (\map key -> M.insertWith (+) key 1 map) M.empty + isRegularEdge (_, _, CFEFlow) = True + isRegularEdge _ = False + + recursiveLookup :: M.Map Node Node -> Node -> Node + recursiveLookup map node = + case M.lookup node map of + Nothing -> node + Just x -> recursiveLookup map x + + isStructural (node, label) = + case label of + CFStructuralNode -> True + _ -> False + + isLinear node = + M.findWithDefault 0 node inDegree == 1 + && M.findWithDefault 0 node outDegree == 1 + + +remapNode :: M.Map Node Node -> LNode CFNode -> LNode CFNode +remapNode m (node, label) = + (remapHelper m node, newLabel) + where + newLabel = case label of + CFApplyEffects effects -> CFApplyEffects (map (remapEffect m) effects) + CFExecuteSubshell s a b -> CFExecuteSubshell s (remapHelper m a) (remapHelper m b) +-- CFSubShellStart reason node -> CFSubShellStart reason (remapHelper m node) + + _ -> label + +remapEffect map old@(IdTagged id effect) = + case effect of + CFDefineFunction name id start end -> IdTagged id $ CFDefineFunction name id (remapHelper map start) (remapHelper map end) + _ -> old + +remapEdge :: M.Map Node Node -> LEdge CFEdge -> LEdge CFEdge +remapEdge map (from, to, label) = (remapHelper map from, remapHelper map to, label) +remapHelper map n = M.findWithDefault n n map + +data Range = Range Node Node + deriving (Eq, Show) + +data CFContext = CFContext { + cfIsCondition :: Bool, + cfIsFunction :: Bool, + cfLoopStack :: [(Node, Node)], + cfExitTarget :: Maybe Node, + cfReturnTarget :: Maybe Node, + cfParameters :: CFGParameters +} +newCFContext params = CFContext { + cfIsCondition = False, + cfIsFunction = False, + cfLoopStack = [], + cfExitTarget = Nothing, + cfReturnTarget = Nothing, + cfParameters = params +} + +-- The monad we generate a graph in +type CFM a = RWS CFContext ([LNode CFNode], [LEdge CFEdge], [(Id, (Node, Node))]) Int a + +newNode :: CFNode -> CFM Node +newNode label = do + n <- get + put (n+1) + tell ([(n, label)], [], []) + return n + +newNodeRange :: CFNode -> CFM Range +-- newNodeRange label = nodeToRange <$> newNode label +newNodeRange label = nodeToRange <$> newNode label + +-- Build a disjoint piece of the graph and return a CFExecuteSubshell. The Id is used purely for debug naming. +subshell :: Id -> String -> CFM Range -> CFM Range +subshell id reason p = do + start <- newNode $ CFEntryPoint $ "Subshell " ++ show id ++ ": " ++ reason + end <- newNode CFStructuralNode + middle <- local (\c -> c { cfExitTarget = Just end, cfReturnTarget = Just end}) p + linkRanges [nodeToRange start, middle, nodeToRange end] + newNodeRange $ CFExecuteSubshell reason start end + + +withFunctionScope p = do + end <- newNode CFStructuralNode + body <- local (\c -> c { cfReturnTarget = Just end, cfIsFunction = True }) p + linkRanges [body, nodeToRange end] + + +nodeToRange :: Node -> Range +nodeToRange n = Range n n + +link :: Node -> Node -> CFEdge -> CFM () +link from to label = do + tell ([], [(from, to, label)], []) + +registerNode :: Id -> Range -> CFM () +registerNode id (Range start end) = tell ([], [], [(id, (start, end))]) + +linkRange :: Range -> Range -> CFM Range +linkRange = linkRangeAs CFEFlow + +linkRangeAs :: CFEdge -> Range -> Range -> CFM Range +linkRangeAs label (Range start mid1) (Range mid2 end) = do + link mid1 mid2 label + return (Range start end) + +-- Like linkRange but without actually linking +spanRange :: Range -> Range -> Range +spanRange (Range start mid1) (Range mid2 end) = Range start end + +linkRanges :: [Range] -> CFM Range +linkRanges [] = error "Empty range" +linkRanges (first:rest) = foldM linkRange first rest + +sequentially :: [Token] -> CFM Range +sequentially list = do + first <- newStructuralNode + rest <- mapM build list + linkRanges (first:rest) + +withContext :: (CFContext -> CFContext) -> CFM a -> CFM a +withContext = local + +withReturn :: Range -> CFM a -> CFM a +withReturn _ p = p + +asCondition :: CFM Range -> CFM Range +asCondition = withContext (\c -> c { cfIsCondition = True }) + +newStructuralNode = newNodeRange CFStructuralNode + +buildRoot :: Token -> CFM Range +buildRoot t = do + entry <- newNodeRange $ CFEntryPoint "MAIN" + impliedExit <- newNode CFImpliedExit + end <- newNode CFStructuralNode + start <- local (\c -> c { cfExitTarget = Just end, cfReturnTarget = Just impliedExit}) $ build t + range <- linkRanges [entry, start, nodeToRange impliedExit, nodeToRange end] + registerNode (getId t) range + return range + +applySingle e = CFApplyEffects [e] + +-- Build the CFG. +build :: Token -> CFM Range +build t = do + range <- build' t + registerNode (getId t) range + return range + where + build' t = case t of + T_Annotation _ _ list -> build list + T_Script _ _ list -> do + sequentially list + + TA_Assignment id op var@(TA_Variable _ name indices) rhs -> do + -- value first: (( var[x=1] = (x=2) )) runs x=1 last + value <- build rhs + subscript <- sequentially indices + read <- + if op == "=" + then none + -- This is += or something + else newNodeRange $ applySingle $ IdTagged id $ CFReadVariable name + + write <- newNodeRange $ applySingle $ IdTagged id $ CFWriteVariable name $ + if null indices + then CFValueInteger + else CFValueArray + + linkRanges [value, subscript, read, write] + + TA_Assignment id op lhs rhs -> do + -- This is likely an invalid assignment like (( 1 = 2 )), but it + -- could be e.g. x=y; (( $x = 3 )); echo $y, so expand both sides + -- without updating anything + sequentially [lhs, rhs] + + TA_Binary _ _ a b -> sequentially [a,b] + TA_Expansion _ list -> sequentially list + TA_Sequence _ list -> sequentially list + + TA_Trinary _ cond a b -> do + condition <- build cond + ifthen <- build a + elsethen <- build b + end <- newStructuralNode + linkRanges [condition, ifthen, end] + linkRanges [condition, elsethen, end] + + TA_Variable id name indices -> do + subscript <- sequentially indices + hint <- + if null indices + then none + else nodeToRange <$> newNode (applySingle $ IdTagged id $ CFHintArray name) + read <- nodeToRange <$> newNode (applySingle $ IdTagged id $ CFReadVariable name) + linkRanges [subscript, hint, read] + + TA_Unary id op (TA_Variable _ name indices) | "--" `isInfixOf` op || "++" `isInfixOf` op -> do + subscript <- sequentially indices + read <- newNodeRange $ applySingle $ IdTagged id $ CFReadVariable name + write <- newNodeRange $ applySingle $ IdTagged id $ CFWriteVariable name $ + if null indices + then CFValueInteger + else CFValueArray + linkRanges [subscript, read, write] + TA_Unary _ _ arg -> build arg + + TC_And _ SingleBracket _ lhs rhs -> do + sequentially [lhs, rhs] + + TC_And _ DoubleBracket _ lhs rhs -> do + left <- build lhs + right <- build rhs + end <- newStructuralNode + -- complete + linkRanges [left, right, end] + -- short circuit + linkRange left end + + -- TODO: Handle integer ops + TC_Binary _ mode str lhs rhs -> do + left <- build lhs + right <- build rhs + linkRange left right + + TC_Empty {} -> newStructuralNode + + TC_Group _ _ t -> build t + + -- TODO: Mark as checked + TC_Nullary _ _ arg -> build arg + + TC_Or _ SingleBracket _ lhs rhs -> sequentially [lhs, rhs] + + TC_Or _ DoubleBracket _ lhs rhs -> do + left <- build lhs + right <- build rhs + end <- newStructuralNode + -- complete + linkRanges [left, right, end] + -- short circuit + linkRange left end + + -- TODO: Handle -v, -z, -n + TC_Unary _ _ op arg -> do + build arg + + T_Arithmetic id root -> do + exe <- build root + status <- newNodeRange (CFSetExitCode id) + linkRange exe status + + T_AndIf _ lhs rhs -> do + left <- build lhs + right <- build rhs + end <- newStructuralNode + linkRange left right + linkRange right end + linkRange left end + + T_Array _ list -> sequentially list + + T_Assignment {} -> buildAssignment DefaultScope t + + T_Backgrounded id body -> do + start <- newStructuralNode + fork <- subshell id "backgrounding '&'" $ build body + pid <- newNodeRange $ CFSetBackgroundPid id + status <- newNodeRange $ CFSetExitCode id + + linkRange start fork + -- Add a join from the fork to warn about variable changes + linkRangeAs CFEFalseFlow fork pid + linkRanges [start, pid, status] + + T_Backticked id body -> + subshell id "`..` expansion" $ sequentially body + + T_Banged id cmd -> do + main <- build cmd + status <- newNodeRange (CFSetExitCode id) + linkRange main status + + T_BatsTest id _ body -> do + -- These are technically set by the 'run' command, but we'll just define them + -- up front to avoid figuring out which commands named "run" belong to Bats. + status <- newNodeRange $ applySingle $ IdTagged id $ CFWriteVariable "status" CFValueInteger + output <- newNodeRange $ applySingle $ IdTagged id $ CFWriteVariable "output" CFValueString + main <- build body + linkRanges [status, output, main] + + T_BraceExpansion _ list -> sequentially list + + T_BraceGroup id body -> + sequentially body + + T_CaseExpression id t [] -> build t + + T_CaseExpression id t list -> do + start <- newStructuralNode + token <- build t + branches <- mapM buildBranch list + end <- newStructuralNode + + let neighbors = zip branches $ tail branches + let (_, firstCond, _) = head branches + let (_, lastCond, lastBody) = last branches + + linkRange start token + linkRange token firstCond + mapM_ (uncurry $ linkBranch end) neighbors + linkRange lastBody end + + unless (any hasCatchAll list) $ + -- There's no *) branch, so assume we can fall through + void $ linkRange token end + + return $ spanRange start end + + where + -- for a | b | c, evaluate each in turn and allow short circuiting + buildCond list = do + start <- newStructuralNode + conds <- mapM build list + end <- newStructuralNode + linkRanges (start:conds) + mapM_ (`linkRange` end) conds + return $ spanRange start end + + buildBranch (typ, cond, body) = do + c <- buildCond cond + b <- sequentially body + linkRange c b + return (typ, c, b) + + linkBranch end (typ, cond, body) (_, nextCond, nextBody) = do + -- Failure case + linkRange cond nextCond + -- After body + case typ of + CaseBreak -> linkRange body end + CaseFallThrough -> linkRange body nextBody + CaseContinue -> linkRange body nextCond + + -- Find a *) if any + + hasCatchAll (_,cond,_) = any isCatchAll cond + isCatchAll c = fromMaybe False $ do + pg <- wordToExactPseudoGlob c + return $ pg `pseudoGlobIsSuperSetof` [PGMany] + + T_Condition _ _ op -> build op + + T_CoProc id maybeName t -> do + let name = fromMaybe "COPROC" maybeName + start <- newStructuralNode + parent <- newNodeRange $ applySingle $ IdTagged id $ CFWriteVariable name CFValueArray + child <- subshell id "coproc" $ build t + end <- newNodeRange $ CFSetExitCode id + + linkRange start parent + linkRange start child + linkRange parent end + linkRangeAs CFEFalseFlow child end + + return $ spanRange start end + T_CoProcBody _ t -> build t + + T_DollarArithmetic _ arith -> build arith + T_DollarDoubleQuoted _ list -> sequentially list + T_DollarSingleQuoted _ _ -> none + T_DollarBracket _ t -> build t + + T_DollarBraced id _ t -> do + let str = concat $ oversimplify t + let modifier = getBracedModifier str + let reference = getBracedReference str + let indices = getIndexReferences str + let offsets = getOffsetReferences str + vals <- build t + others <- mapM (\x -> nodeToRange <$> newNode (applySingle $ IdTagged id $ CFReadVariable x)) (indices ++ offsets) + deps <- linkRanges (vals:others) + read <- nodeToRange <$> newNode (applySingle $ IdTagged id $ CFReadVariable reference) + totalRead <- linkRange deps read + + if any (`isPrefixOf` modifier) ["=", ":="] + then do + optionalAssign <- newNodeRange (applySingle $ IdTagged id $ CFWriteVariable reference CFValueString) + result <- newStructuralNode + linkRange optionalAssign result + linkRange totalRead result + else return totalRead + + T_DoubleQuoted _ list -> sequentially list + + T_DollarExpansion id body -> + subshell id "$(..) expansion" $ sequentially body + + T_Extglob _ _ list -> sequentially list + + T_FdRedirect id ('{':identifier) op -> do + let name = takeWhile (/= '}') identifier + expression <- build op + rw <- newNodeRange $ + if isClosingFileOp op + then applySingle $ IdTagged id $ CFReadVariable name + else applySingle $ IdTagged id $ CFWriteVariable name CFValueInteger + + linkRange expression rw + + + T_FdRedirect _ name t -> do + build t + + T_ForArithmetic _ initT condT incT bodyT -> do + init <- build initT + cond <- build condT + body <- sequentially bodyT + inc <- build incT + end <- newStructuralNode + + -- Forward edges + linkRanges [init, cond, body, inc] + linkRange cond end + -- Backward edge + linkRange inc cond + return $ spanRange init end + + T_ForIn id name words body -> forInHelper id name words body + + -- For functions we generate an unlinked subgraph, and mention that in its definition node + T_Function id _ _ name body -> do + range <- local (\c -> c { cfExitTarget = Nothing }) $ do + entry <- newNodeRange $ CFEntryPoint $ "function " ++ name + f <- withFunctionScope $ build body + linkRange entry f + let (Range entry exit) = range + definition <- newNodeRange (applySingle $ IdTagged id $ CFDefineFunction name id entry exit) + exe <- newNodeRange (CFSetExitCode id) + linkRange definition exe + + T_Glob {} -> none + + T_HereString _ t -> build t + T_HereDoc _ _ _ _ list -> sequentially list + + T_IfExpression id ifs elses -> do + start <- newStructuralNode + branches <- doBranches start ifs elses [] + end <- newStructuralNode + mapM_ (`linkRange` end) branches + return $ spanRange start end + where + doBranches start ((conds, thens):rest) elses result = do + cond <- asCondition $ sequentially conds + action <- sequentially thens + linkRange start cond + linkRange cond action + doBranches cond rest elses (action:result) + doBranches start [] elses result = do + rest <- + if null elses + then newNodeRange (CFSetExitCode id) + else sequentially elses + linkRange start rest + return (rest:result) + + T_Include _ t -> build t + + T_IndexedElement _ indicesT valueT -> do + indices <- sequentially indicesT + value <- build valueT + linkRange indices value + + T_IoDuplicate _ op _ -> build op + + T_IoFile _ op t -> do + exp <- build t + doesntDoMuch <- build op + linkRange exp doesntDoMuch + + T_Literal {} -> none + + T_NormalWord _ list -> sequentially list + + T_OrIf _ lhs rhs -> do + left <- build lhs + right <- build rhs + end <- newStructuralNode + linkRange left right + linkRange right end + linkRange left end + + T_Pipeline _ _ [cmd] -> build cmd + T_Pipeline id _ cmds -> do + start <- newStructuralNode + hasLastpipe <- reader $ cfLastpipe . cfParameters + (leading, last) <- buildPipe hasLastpipe cmds + end <- newStructuralNode + + mapM_ (linkRange start) leading + mapM_ (\c -> linkRangeAs CFEFalseFlow c end) leading + linkRanges $ [start] ++ last ++ [end] + where + buildPipe True [x] = do + last <- build x + return ([], [last]) + buildPipe lp (first:rest) = do + this <- subshell id "pipeline" $ build first + (leading, last) <- buildPipe lp rest + return (this:leading, last) + buildPipe _ [] = return ([], []) + + T_ProcSub id op cmds -> do + start <- newStructuralNode + body <- subshell id (op ++ "() process substitution") $ sequentially cmds + end <- newStructuralNode + + linkRange start body + linkRangeAs CFEFalseFlow body end + linkRange start end + + T_Redirecting _ redirs cmd -> do + -- For simple commands, this is the other way around in bash + -- We do it in this order for comound commands like { x=name; } > "$x" + redir <- sequentially redirs + body <- build cmd + linkRange redir body + + T_SelectIn id name words body -> forInHelper id name words body + + T_SimpleCommand id vars [] -> do + -- Vars can also be empty, as in the command "> foo" + assignments <- sequentially vars + status <- newNodeRange (CFSetExitCode id) + linkRange assignments status + + T_SimpleCommand id vars list@(cmd:_) -> + handleCommand t vars list $ getUnquotedLiteral cmd + + T_SingleQuoted _ _ -> none + + T_SourceCommand _ originalCommand inlinedSource -> do + cmd <- build originalCommand + end <- newStructuralNode + inline <- withReturn end $ build inlinedSource + linkRange cmd inline + linkRange inline end + return $ spanRange cmd inline + + T_Subshell id body -> do + main <- subshell id "explicit (..) subshell" $ sequentially body + status <- newNodeRange (CFSetExitCode id) + linkRange main status + + T_UntilExpression id cond body -> whileHelper id cond body + T_WhileExpression id cond body -> whileHelper id cond body + + T_CLOBBER _ -> none + T_GREATAND _ -> none + T_LESSAND _ -> none + T_LESSGREAT _ -> none + T_DGREAT _ -> none + T_Greater _ -> none + T_Less _ -> none + T_ParamSubSpecialChar _ _ -> none + + x -> error ("Unimplemented: " ++ show x) + +-- Still in `where` clause + forInHelper id name words body = do + entry <- newStructuralNode + expansion <- sequentially words + assignmentChoice <- newStructuralNode + assignments <- + if null words || any willSplit words + then (:[]) <$> (newNodeRange $ applySingle $ IdTagged id $ CFWriteVariable name CFValueString) + else mapM (\t -> newNodeRange $ applySingle $ IdTagged id $ CFWriteVariable name $ CFValueComputed (getId t) $ tokenToParts t) words + body <- sequentially body + exit <- newStructuralNode + -- Forward edges + linkRanges [entry, expansion, assignmentChoice] + mapM_ (\t -> linkRanges [assignmentChoice, t, body]) assignments + linkRange body exit + linkRange expansion exit + -- Backward edge + linkRange body assignmentChoice + return $ spanRange entry exit + + whileHelper id cond body = do + condRange <- asCondition $ sequentially cond + bodyRange <- sequentially body + end <- newNodeRange (CFSetExitCode id) + + linkRange condRange bodyRange + linkRange bodyRange condRange + linkRange condRange end + + +handleCommand cmd vars args literalCmd = do + -- TODO: Handle assignments in declaring commands + + case literalCmd of + Just "exit" -> regularExpansion vars args $ handleExit + Just "return" -> regularExpansion vars args $ handleReturn + Just "unset" -> regularExpansionWithStatus vars args $ handleUnset args + + Just "declare" -> handleDeclare args + Just "local" -> handleDeclare args + Just "typeset" -> handleDeclare args + + Just "printf" -> regularExpansionWithStatus vars args $ handlePrintf args + Just "wait" -> regularExpansionWithStatus vars args $ handleWait args + + Just "mapfile" -> regularExpansionWithStatus vars args $ handleMapfile args + Just "readarray" -> regularExpansionWithStatus vars args $ handleMapfile args + + Just "DEFINE_boolean" -> regularExpansionWithStatus vars args $ handleDEFINE args + Just "DEFINE_float" -> regularExpansionWithStatus vars args $ handleDEFINE args + Just "DEFINE_integer" -> regularExpansionWithStatus vars args $ handleDEFINE args + Just "DEFINE_string" -> regularExpansionWithStatus vars args $ handleDEFINE args + + -- This will mostly behave like 'command' but ok + Just "builtin" -> + case args of + [_] -> regular + (_:newargs@(newcmd:_)) -> + handleCommand newcmd vars newargs $ getLiteralString newcmd + Just "command" -> + case args of + [_] -> regular + (_:newargs@(newcmd:_)) -> + handleOthers (getId newcmd) vars newargs $ getLiteralString newcmd + _ -> regular + + where + regular = handleOthers (getId cmd) vars args literalCmd + handleExit = do + exitNode <- reader cfExitTarget + case exitNode of + Just target -> do + exit <- newNode CFResolvedExit + link exit target CFEExit + unreachable <- newNode CFUnreachable + return $ Range exit unreachable + Nothing -> do + exit <- newNode CFUnresolvedExit + unreachable <- newNode CFUnreachable + return $ Range exit unreachable + + handleReturn = do + returnTarget <- reader cfReturnTarget + case returnTarget of + Nothing -> error $ pleaseReport "missing return target" + Just target -> do + ret <- newNode CFStructuralNode + link ret target CFEFlow + unreachable <- newNode CFUnreachable + return $ Range ret unreachable + + handleUnset (cmd:args) = do + case () of + _ | "n" `elem` flagNames -> unsetWith CFUndefineNameref + _ | "v" `elem` flagNames -> unsetWith CFUndefineVariable + _ | "f" `elem` flagNames -> unsetWith CFUndefineFunction + _ -> unsetWith CFUndefine + where + pairs :: [(String, Token)] -- [(Flag string, token)] e.g. [("-f", t), ("", myfunc)] + pairs = map (\(str, (flag, val)) -> (str, flag)) $ fromMaybe (map (\c -> ("", (c,c))) args) $ getGnuOpts "vfn" args + (names, flags) = partition (null . fst) pairs + flagNames = map fst flags + literalNames :: [(Token, String)] -- Literal names to unset, e.g. [(myfuncToken, "myfunc")] + literalNames = mapMaybe (\(_, t) -> getLiteralString t >>= (return . (,) t)) names + -- Apply a constructor like CFUndefineVariable to each literalName, and tag with its id + unsetWith c = newNodeRange $ CFApplyEffects $ map (\(token, name) -> IdTagged (getId token) $ c name) literalNames + + + variableAssignRegex = mkRegex "^([_a-zA-Z][_a-zA-Z0-9]*)=" + + handleDeclare (cmd:args) = do + isFunc <- asks cfIsFunction + let (evaluated, effects) = mconcat $ map (toEffects isFunc) args + before <- sequentially $ evaluated + effect <- newNodeRange $ CFApplyEffects effects + result <- newNodeRange $ CFSetExitCode (getId cmd) + linkRanges [before, effect, result] + where + opts = map fst $ getGenericOpts args + array = "a" `elem` opts || "A" `elem` opts + integer = "i" `elem` opts + func = "f" `elem` opts || "F" `elem` opts + global = "g" `elem` opts + writer isFunc = + case () of + _ | global -> CFWriteGlobal + _ | isFunc -> CFWriteLocal + _ -> CFWriteVariable + + toEffects :: Bool -> Token -> ([Token], [IdTagged CFEffect]) + toEffects isFunc (T_Assignment id mode var idx t) = + let + pre = idx ++ [t] + isArray = array || (not $ null idx) + asArray = [ IdTagged id $ (writer isFunc) var CFValueArray ] + asString = [ IdTagged id $ (writer isFunc) var $ + if integer + then CFValueInteger -- TODO: Also handle integer variable property + else CFValueComputed (getId t) $ [ CFStringVariable var | mode == Append ] ++ tokenToParts t + ] + in + (pre, if isArray then asArray else asString ) + + toEffects isFunc t = + let + pre = [t] + literal = fromJust $ getLiteralStringExt (const $ Just "\0") t + isKnown = '\0' `notElem` literal + match = fmap head $ variableAssignRegex `matchRegex` literal + name = fromMaybe literal match + + typer def = + if array + then CFValueArray + else + if integer + then CFValueInteger + else def + + asLiteral = [ + IdTagged (getId t) $ (writer isFunc) name $ + typer $ CFValueComputed (getId t) [ CFStringLiteral $ drop 1 $ dropWhile (/= '=') $ literal ] + ] + asUnknown = [ + IdTagged (getId t) $ (writer isFunc) name $ + typer $ CFValueString + ] + asBlank = [ + IdTagged (getId t) $ (writer isFunc) name $ + typer $ CFValueComputed (getId t) [] + ] + in + case () of + _ | not (isVariableName name) -> (pre, []) + _ | isJust match && isKnown -> (pre, asLiteral) + _ | isJust match -> (pre, asUnknown) + _ -> (pre, asBlank) + + handlePrintf (cmd:args) = + newNodeRange $ CFApplyEffects $ maybeToList findVar + where + findVar = do + flags <- getBsdOpts "v:" args + (flag, arg) <- lookup "v" flags + name <- getLiteralString arg + return $ IdTagged (getId arg) $ CFWriteVariable name CFValueString + + handleWait (cmd:args) = + newNodeRange $ CFApplyEffects $ maybeToList findVar + where + findVar = do + let flags = getGenericOpts args + (flag, arg) <- lookup "p" flags + name <- getLiteralString arg + return $ IdTagged (getId arg) $ CFWriteVariable name CFValueInteger + + handleMapfile (cmd:args) = + newNodeRange $ CFApplyEffects [findVar] + where + findVar = + let (id, name) = fromMaybe (getId cmd, "MAPFILE") $ getFromArg `mplus` getFromFallback + in IdTagged id $ CFWriteVariable name CFValueArray + + getFromArg = do + flags <- getGnuOpts "d:n:O:s:u:C:c:t" args + (_, arg) <- lookup "" flags + name <- getLiteralString arg + return (getId arg, name) + + getFromFallback = + listToMaybe $ mapMaybe getIfVar $ reverse args + getIfVar c = do + name <- getLiteralString c + guard $ isVariableName name + return (getId c, name) + + handleDEFINE (cmd:args) = + newNodeRange $ CFApplyEffects $ maybeToList findVar + where + findVar = do + name <- listToMaybe $ drop 1 args + str <- getLiteralString name + guard $ isVariableName str + return $ IdTagged (getId name) $ CFWriteVariable str CFValueString + + handleOthers id vars args cmd = + regularExpansion vars args $ do + exe <- newNodeRange $ CFExecuteCommand cmd + status <- newNodeRange $ CFSetExitCode id + linkRange exe status + + regularExpansion vars args p = do + args <- sequentially args + assignments <- mapM (buildAssignment PrefixScope) vars + exe <- p + dropAssignments <- + if null vars + then + return [] + else do + drop <- newNodeRange CFDropPrefixAssignments + return [drop] + + linkRanges $ [args] ++ assignments ++ [exe] ++ dropAssignments + + regularExpansionWithStatus vars args@(cmd:_) p = do + initial <- regularExpansion vars args p + status <- newNodeRange $ CFSetExitCode (getId cmd) + linkRange initial status + + +none = newStructuralNode + +data Scope = DefaultScope | GlobalScope | LocalScope | PrefixScope + +buildAssignment scope t = do + op <- case t of + T_Assignment id mode var indices value -> do + expand <- build value + index <- sequentially indices + read <- case mode of + Append -> newNodeRange (applySingle $ IdTagged id $ CFReadVariable var) + Assign -> none + let valueType = if null indices then f id value else CFValueArray + let scoper = + case scope of + PrefixScope -> CFWritePrefix + LocalScope -> CFWriteLocal + GlobalScope -> CFWriteGlobal + DefaultScope -> CFWriteVariable + write <- newNodeRange $ applySingle $ IdTagged id $ scoper var valueType + linkRanges [expand, index, read, write] + where + f :: Id -> Token -> CFValue + f id t@T_NormalWord {} = CFValueComputed id $ [CFStringVariable var | mode == Append] ++ tokenToParts t + f id t@(T_Literal _ str) = CFValueComputed id $ [CFStringVariable var | mode == Append] ++ tokenToParts t + f _ T_Array {} = CFValueArray + + registerNode (getId t) op + return op + + +tokenToParts t = + case t of + T_NormalWord _ list -> concatMap tokenToParts list + T_DoubleQuoted _ list -> concatMap tokenToParts list + T_SingleQuoted _ str -> [ CFStringLiteral str ] + T_Literal _ str -> [ CFStringLiteral str ] + T_DollarArithmetic {} -> [ CFStringInteger ] + T_DollarBracket {} -> [ CFStringInteger ] + T_DollarBraced _ _ list | isUnmodifiedParameterExpansion t -> [ CFStringVariable (getBracedReference $ concat $ oversimplify list) ] + -- Check if getLiteralString can handle it, if not it's unknown + _ -> [maybe CFStringUnknown CFStringLiteral $ getLiteralString t] + +return [] +runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) diff --git a/src/ShellCheck/CFGAnalysis.hs b/src/ShellCheck/CFGAnalysis.hs new file mode 100644 index 0000000..99ce450 --- /dev/null +++ b/src/ShellCheck/CFGAnalysis.hs @@ -0,0 +1,1113 @@ +{- + Copyright 2022 Vidar Holen + + This file is part of ShellCheck. + https://www.shellcheck.net + + ShellCheck is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + ShellCheck is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +-} +{-# LANGUAGE TemplateHaskell #-} +{-# LANGUAGE RankNTypes #-} +{-# LANGUAGE DeriveAnyClass, DeriveGeneric #-} + +{- + Data Flow Analysis on a Control Flow Graph. + + This module implements a pretty standard iterative Data Flow Analysis. + For an overview of the process, see Wikipedia. + + Since shell scripts rely heavily on global variables, this DFA includes + tracking the value of globals across calls. Each function invocation is + treated as a separate DFA problem, and a caching mechanism (hopefully) + avoids any exponential explosions. + + To do efficient DFA join operations (or merges, as the code calls them), + some of the data structures have an integer version attached. On update, + the version is changed. If two states have the same version number, + a merge is skipped on the grounds that they are identical. It is easy + to unintentionally forget to update/invalidate the version number, + and bugs will ensure. + + For performance reasons, the entire code runs in plain ST, with a manual + context object Ctx being passed around. It relies heavily on mutable + STRefs. However, this turned out to be literally thousands of times faster + than my several attempts using RWST, so it can't be helped. +-} + +module ShellCheck.CFGAnalysis ( + analyzeControlFlow + ,CFGParameters (..) + ,CFGAnalysis (..) + ,ProgramState (..) + ,VariableValue (..) + ,SpaceStatus (..) + ,getIncomingState + ,getOutgoingState + ,ShellCheck.CFGAnalysis.runTests -- STRIP + ) where + +import GHC.Generics (Generic) +import ShellCheck.AST +import ShellCheck.CFG +import qualified ShellCheck.Data as Data +import ShellCheck.Prelude +import Control.Monad +import Control.Monad.ST +import Control.DeepSeq +import Data.List hiding (map) +import Data.STRef +import Data.Maybe +import qualified Data.Map as M +import qualified Data.Set as S +import Data.Graph.Inductive.Graph +import Data.Graph.Inductive.Query.DFS +import Debug.Trace -- STRIP + +import Test.QuickCheck + + +iterationCount = 1000000 +cacheEntries = 10 + +-- The result of the data flow analysis +data CFGAnalysis = CFGAnalysis { + graph :: CFGraph, + tokenToNode :: M.Map Id (Node, Node), + nodeToData :: M.Map Node (ProgramState, ProgramState) +} deriving (Show, Generic, NFData) + +-- The program state we expose externally +data ProgramState = ProgramState { + variablesInScope :: M.Map String VariableValue, + stateIsReachable :: Bool +-- internalState :: InternalState +} deriving (Show, Eq, Generic, NFData) + +-- Conveniently get the state before a token id +getIncomingState :: CFGAnalysis -> Id -> Maybe ProgramState +getIncomingState analysis id = do + (start,end) <- M.lookup id $ tokenToNode analysis + fst <$> M.lookup start (nodeToData analysis) + +-- Conveniently get the state after a token id +getOutgoingState :: CFGAnalysis -> Id -> Maybe ProgramState +getOutgoingState analysis id = do + (start,end) <- M.lookup id $ tokenToNode analysis + snd <$> M.lookup end (nodeToData analysis) + +getDataForNode analysis node = M.lookup node $ nodeToData analysis + +-- The current state of data flow at a point in the program, potentially as a diff +data InternalState = InternalState { + sVersion :: Integer, + sGlobalValues :: VersionedMap String VariableValue, + sLocalValues :: VersionedMap String VariableValue, + sPrefixValues :: VersionedMap String VariableValue, + sFunctionTargets :: VersionedMap String FunctionValue, + sIsReachable :: Maybe Bool +} deriving (Show, Generic, NFData) + +newInternalState = InternalState { + sVersion = 0, + sGlobalValues = vmEmpty, + sLocalValues = vmEmpty, + sPrefixValues = vmEmpty, + sFunctionTargets = vmEmpty, + sIsReachable = Nothing +} + +unreachableState = modified newInternalState { + sIsReachable = Just False +} + +-- The default state we assume we get from the environment +createEnvironmentState :: InternalState +createEnvironmentState = do + foldl' (flip ($)) newInternalState $ concat [ + addVars Data.internalVariables unknownVariableValue, + addVars Data.variablesWithoutSpaces spacelessVariableValue, + addVars Data.specialIntegerVariables spacelessVariableValue + ] + where + addVars names val = map (\name -> insertGlobal name val) names + spacelessVariableValue = VariableValue { + literalValue = Nothing, + spaceStatus = SpaceStatusClean + } + + +modified s = s { sVersion = -1 } + +insertGlobal :: String -> VariableValue -> InternalState -> InternalState +insertGlobal name value state = modified state { + sGlobalValues = vmInsert name value $ sGlobalValues state +} + +insertLocal :: String -> VariableValue -> InternalState -> InternalState +insertLocal name value state = modified state { + sLocalValues = vmInsert name value $ sLocalValues state +} + +insertPrefix :: String -> VariableValue -> InternalState -> InternalState +insertPrefix name value state = modified state { + sPrefixValues = vmInsert name value $ sPrefixValues state +} + +insertFunction :: String -> FunctionValue -> InternalState -> InternalState +insertFunction name value state = modified state { + sFunctionTargets = vmInsert name value $ sFunctionTargets state +} + +internalToExternal :: InternalState -> ProgramState +internalToExternal s = + ProgramState { + -- Avoid introducing dependencies on the literal value as this is only for debugging purposes right now + variablesInScope = M.map (\c -> c { literalValue = Nothing }) flatVars, + -- internalState = s, -- For debugging + stateIsReachable = fromMaybe True $ sIsReachable s + } + where + flatVars = M.unionsWith (\_ last -> last) $ map mapStorage [sGlobalValues s, sLocalValues s, sPrefixValues s] + +-- Dependencies on values, e.g. "if there is a global variable named 'foo' without spaces" +-- This is used to see if the DFA of a function would result in the same state, so anything +-- that affects DFA must be tracked. +data StateDependency = + DepGlobalValue String VariableValue + | DepLocalValue String VariableValue + | DepPrefixValue String VariableValue + | DepFunction String (S.Set FunctionDefinition) + -- Whether invoking the node would result in recursion (i.e., is the function on the stack?) + | DepIsRecursive Node Bool + deriving (Show, Eq, Ord, Generic, NFData) + +-- A function definition, or lack thereof +data FunctionDefinition = FunctionUnknown | FunctionDefinition String Node Node + deriving (Show, Eq, Ord, Generic, NFData) + +-- The Set of places a command name can point (it's a Set to handle conditionally defined functions) +type FunctionValue = S.Set FunctionDefinition + +-- The scope of a function. ("Prefix" refers to e.g. `foo=1 env`) +data VariableScope = PrefixVar | LocalVar | GlobalVar + deriving (Show, Eq, Ord, Generic, NFData) + +-- Create an InternalState that fulfills the given dependencies +depsToState :: S.Set StateDependency -> InternalState +depsToState set = foldl insert newInternalState $ S.toList set + where + insert :: InternalState -> StateDependency -> InternalState + insert state dep = + case dep of + DepFunction name val -> insertFunction name val state + DepGlobalValue name val -> insertGlobal name val state + DepLocalValue name val -> insertLocal name val state + DepPrefixValue name val -> insertPrefix name val state + DepIsRecursive _ _ -> state + +unknownFunctionValue = S.singleton FunctionUnknown + +-- The information about the value of a single variable +data VariableValue = VariableValue { + literalValue :: Maybe String, -- TODO: For debugging. Remove me. + spaceStatus :: SpaceStatus +} + deriving (Show, Eq, Ord, Generic, NFData) + +-- Whether or not the value needs quoting (has spaces/globs), or we don't know +data SpaceStatus = SpaceStatusEmpty | SpaceStatusClean | SpaceStatusDirty deriving (Show, Eq, Ord, Generic, NFData) + + +unknownVariableValue = VariableValue { + literalValue = Nothing, + spaceStatus = SpaceStatusDirty +} + +emptyVariableValue = VariableValue { + literalValue = Just "", + spaceStatus = SpaceStatusEmpty +} + +mergeVariableValue a b = VariableValue { + literalValue = if literalValue a == literalValue b then literalValue a else Nothing, + spaceStatus = mergeSpaceStatus (spaceStatus a) (spaceStatus b) +} + +mergeSpaceStatus a b = + case (a,b) of + (SpaceStatusEmpty, y) -> y + (x, SpaceStatusEmpty) -> x + (SpaceStatusClean, SpaceStatusClean) -> SpaceStatusClean + _ -> SpaceStatusDirty + +-- A VersionedMap is a Map that keeps an additional integer version to quickly determine if it has changed. +-- * Version -1 means it's unknown (possibly and presumably changed) +-- * Version 0 means it's empty +-- * Version N means it's equal to any other map with Version N (this is required but not enforced) +data VersionedMap k v = VersionedMap { + mapVersion :: Integer, + mapStorage :: M.Map k v +} + deriving (Generic, NFData) + +-- This makes states more readable but inhibits copy-paste +instance (Show k, Show v) => Show (VersionedMap k v) where + show m = (if mapVersion m >= 0 then "V" ++ show (mapVersion m) else "U") ++ " " ++ show (mapStorage m) + +instance Eq InternalState where + (==) a b = stateIsQuickEqual a b || stateIsSlowEqual a b + +instance (Eq k, Eq v) => Eq (VersionedMap k v) where + (==) a b = vmIsQuickEqual a b || mapStorage a == mapStorage b + +instance (Ord k, Ord v) => Ord (VersionedMap k v) where + compare a b = + if vmIsQuickEqual a b + then EQ + else mapStorage a `compare` mapStorage b + + +-- A context with STRefs manually passed around to function. +-- This is done because it was dramatically much faster than any RWS type stack +data Ctx s = Ctx { + -- The current node + cNode :: STRef s Node, + -- The current input state + cInput :: STRef s InternalState, + -- The current output state + cOutput :: STRef s InternalState, + + -- The current functions/subshells stack + cStack :: [StackEntry s], + -- The input graph + cGraph :: CFGraph, + -- An incrementing counter to version maps + cCounter :: STRef s Integer, + -- A cache of input state dependencies to output effects + cCache :: STRef s (M.Map Node [(S.Set StateDependency, InternalState)]), + -- The states resulting from data flows per invocation path + cInvocations :: STRef s (M.Map [Node] (S.Set StateDependency, M.Map Node (InternalState, InternalState))) +} + +-- Whenever a function (or subshell) is invoked, a value like this is pushed onto the stack +data StackEntry s = StackEntry { + -- The entry point of this stack entry for the purpose of detecting recursion + entryPoint :: Node, + -- The node where this entry point was invoked + callSite :: Node, + -- A mutable set of dependencies we fetched from here or higher in the stack + dependencies :: STRef s (S.Set StateDependency), + -- The original input state for this stack entry + stackState :: InternalState +} + deriving (Eq, Generic, NFData) + + +-- Overwrite a base state with the contents of a diff state +-- This is unrelated to join/merge. +patchState :: InternalState -> InternalState -> InternalState +patchState base diff = + case () of + _ | sVersion diff == 0 -> base + _ | sVersion base == 0 -> diff + _ | stateIsQuickEqual base diff -> diff + _ -> + InternalState { + sVersion = -1, + sGlobalValues = vmPatch (sGlobalValues base) (sGlobalValues diff), + sLocalValues = vmPatch (sLocalValues base) (sLocalValues diff), + sPrefixValues = vmPatch (sPrefixValues base) (sPrefixValues diff), + sFunctionTargets = vmPatch (sFunctionTargets base) (sFunctionTargets diff), + sIsReachable = sIsReachable diff `mplus` sIsReachable base + } + +patchOutputM ctx diff = do + let cOut = cOutput ctx + oldState <- readSTRef cOut + let newState = patchState oldState diff + writeSTRef cOut newState + +-- Merge (aka Join) two states. This is monadic because it requires looking up +-- values from the current context. For example: +-- +-- f() { +-- foo || x=2 +-- HERE # This merge requires looking up the value of $x in the parent frame +-- } +-- x=1 +-- f +mergeState :: forall s. Ctx s -> InternalState -> InternalState -> ST s InternalState +mergeState ctx a b = do + -- Kludge: we want `readVariable` & friends not to read from an intermediate state, + -- so temporarily set a blank input. + let cin = cInput ctx + old <- readSTRef cin + writeSTRef cin newInternalState + x <- merge a b + writeSTRef cin old + return x + + where + + merge a b = + case () of + _ | sIsReachable a == Just True && sIsReachable b == Just False + || sIsReachable a == Just False && sIsReachable b == Just True -> + error $ pleaseReport "Unexpected merge of reachable and unreachable state" + _ | sIsReachable a == Just False && sIsReachable b == Just False -> + return unreachableState + _ | sVersion a >= 0 && sVersion b >= 0 && sVersion a == sVersion b -> return a + _ -> do + globals <- mergeMaps ctx mergeVariableValue readGlobal (sGlobalValues a) (sGlobalValues b) + locals <- mergeMaps ctx mergeVariableValue readVariable (sLocalValues a) (sLocalValues b) + prefix <- mergeMaps ctx mergeVariableValue readVariable (sPrefixValues a) (sPrefixValues b) + funcs <- mergeMaps ctx S.union readFunction (sFunctionTargets a) (sFunctionTargets b) + return $ InternalState { + sVersion = -1, + sGlobalValues = globals, + sLocalValues = locals, + sPrefixValues = prefix, + sFunctionTargets = funcs, + sIsReachable = liftM2 (&&) (sIsReachable a) (sIsReachable b) + } + +-- Merge a number of states, or return a default if there are no states +-- (it can't fold from newInternalState because this would be equivalent of adding a new input edge). +mergeStates :: forall s. Ctx s -> InternalState -> [InternalState] -> ST s InternalState +mergeStates ctx def list = + case list of + [] -> return def + (first:rest) -> foldM (mergeState ctx) first rest + +-- Merge two maps, key by key. If both maps have a key, the 'merger' is used. +-- If only one has the key, the 'reader' is used to fetch a second, and the two are merged as above. +mergeMaps :: (Ord k) => forall s. + Ctx s -> + (v -> v -> v) -> + (Ctx s -> k -> ST s v) -> + (VersionedMap k v) -> + (VersionedMap k v) -> + ST s (VersionedMap k v) +mergeMaps ctx merger reader a b = + if vmIsQuickEqual a b + then return a + else do + new <- M.fromDistinctAscList <$> reverse <$> f [] (M.toAscList $ mapStorage a) (M.toAscList $ mapStorage b) + vmFromMap ctx new + where + f l [] [] = return l + f l [] b = f l b [] + f l ((k,v):rest1) [] = do + other <- reader ctx k + f ((k, merger v other):l) rest1 [] + f l l1@((k1, v1):rest1) l2@((k2, v2):rest2) = + case k1 `compare` k2 of + EQ -> + f ((k1, merger v1 v2):l) rest1 rest2 + LT -> do + nv2 <- reader ctx k1 + f ((k1, merger v1 nv2):l) rest1 l2 + GT -> do + nv1 <- reader ctx k2 + f ((k2, merger nv1 v2):l) l1 rest2 + +vmFromMap ctx map = return $ VersionedMap { + mapVersion = -1, + mapStorage = map +} + +-- Give a VersionedMap a version if it does not already have one. +versionMap ctx map = + if mapVersion map >= 0 + then return map + else do + v <- nextVersion ctx + return map { + mapVersion = v + } + +-- Give an InternalState a version if it does not already have one. +versionState ctx state = + if sVersion state >= 0 + then return state + else do + self <- nextVersion ctx + ssGlobalValues <- versionMap ctx $ sGlobalValues state + ssLocalValues <- versionMap ctx $ sLocalValues state + ssFunctionTargets <- versionMap ctx $ sFunctionTargets state + return state { + sVersion = self, + sGlobalValues = ssGlobalValues, + sLocalValues = ssLocalValues, + sFunctionTargets = ssFunctionTargets + } + +-- Like 'not null' but for 2+ elements +is2plus :: [a] -> Bool +is2plus l = case l of + _:_:_ -> True + _ -> False + +-- Use versions to see if two states are trivially identical +stateIsQuickEqual a b = + let + va = sVersion a + vb = sVersion b + in + va >= 0 && vb >= 0 && va == vb + +-- A manual slow path 'Eq' (it's not derived because it's part of the custom Eq instance) +stateIsSlowEqual a b = + check sGlobalValues + && check sLocalValues + && check sPrefixValues + && check sFunctionTargets + && check sIsReachable + where + check f = f a == f b + +-- Check if two VersionedMaps are trivially equal +vmIsQuickEqual :: VersionedMap k v -> VersionedMap k v -> Bool +vmIsQuickEqual a b = + let + va = mapVersion a + vb = mapVersion b + in + va >= 0 && vb >= 0 && va == vb + +-- A new, empty VersionedMap +vmEmpty = VersionedMap { + mapVersion = 0, + mapStorage = M.empty +} + +-- Map.null for VersionedMaps +vmNull :: VersionedMap k v -> Bool +vmNull m = mapVersion m == 0 || (M.null $ mapStorage m) + +-- Map.lookup for VersionedMaps +vmLookup name map = M.lookup name $ mapStorage map + +-- Map.insert for VersionedMaps +vmInsert key val map = VersionedMap { + mapVersion = -1, + mapStorage = M.insert key val $ mapStorage map +} + +-- Overwrite all keys in the first map with values from the second +vmPatch :: (Ord k) => VersionedMap k v -> VersionedMap k v -> VersionedMap k v +vmPatch base diff = + case () of + _ | mapVersion base == 0 -> diff + _ | mapVersion diff == 0 -> base + _ | vmIsQuickEqual base diff -> diff + _ -> VersionedMap { + mapVersion = -1, + mapStorage = M.unionWith (flip const) (mapStorage base) (mapStorage diff) + } + +-- Modify a variable as with x=1. This applies it to the appropriate scope. +writeVariable :: forall s. Ctx s -> String -> VariableValue -> ST s () +writeVariable ctx name val = do + (_, typ) <- readVariableWithScope ctx name + case typ of + GlobalVar -> writeGlobal ctx name val + LocalVar -> writeLocal ctx name val + -- Prefixed variables actually become local variables in the invoked function + PrefixVar -> writeLocal ctx name val + +writeGlobal ctx name val = do + modifySTRef (cOutput ctx) $ insertGlobal name val + +writeLocal ctx name val = do + modifySTRef (cOutput ctx) $ insertLocal name val + +writePrefix ctx name val = do + modifySTRef (cOutput ctx) $ insertPrefix name val + +-- Look up a variable value, and also return its scope +readVariableWithScope :: forall s. Ctx s -> String -> ST s (VariableValue, VariableScope) +readVariableWithScope ctx name = lookupStack get dep def ctx name + where + def = (unknownVariableValue, GlobalVar) + get = getVariableWithScope + dep k v = + case v of + (val, GlobalVar) -> DepGlobalValue k val + (val, LocalVar) -> DepLocalValue k val + (val, PrefixVar) -> DepPrefixValue k val + +getVariableWithScope :: InternalState -> String -> Maybe (VariableValue, VariableScope) +getVariableWithScope s name = + case (vmLookup name $ sPrefixValues s, vmLookup name $ sLocalValues s, vmLookup name $ sGlobalValues s) of + (Just var, _, _) -> return (var, PrefixVar) + (_, Just var, _) -> return (var, LocalVar) + (_, _, Just var) -> return (var, GlobalVar) + _ -> Nothing + +undefineFunction ctx name = + writeFunction ctx name $ FunctionUnknown + +undefineVariable ctx name = + writeVariable ctx name $ emptyVariableValue + +readVariable ctx name = fst <$> readVariableWithScope ctx name + +readGlobal ctx name = lookupStack get dep def ctx name + where + def = unknownVariableValue + get s name = vmLookup name $ sGlobalValues s + dep k v = DepGlobalValue k v + +readFunction ctx name = lookupStack get dep def ctx name + where + def = unknownFunctionValue + get s name = vmLookup name $ sFunctionTargets s + dep k v = DepFunction k v + +writeFunction ctx name val = do + modifySTRef (cOutput ctx) $ insertFunction name $ S.singleton val + +-- Look up each state on the stack until a value is found (or the default is used), +-- then add this value as a StateDependency. +lookupStack :: forall s k v. + -- A function that maybe finds a value from a state + (InternalState -> k -> Maybe v) + -- A function that creates a dependency on what was found + -> (k -> v -> StateDependency) + -- A default value, if the value can't be found anywhere + -> v + -- Context + -> Ctx s + -- The key to look up + -> k + -- Returning the result + -> ST s v +lookupStack get dep def ctx key = do + top <- readSTRef $ cInput ctx + case get top key of + Just v -> return v + Nothing -> f (cStack ctx) + where + f [] = return def + f (s:rest) = do + -- Go up the stack until we find the value, and add + -- a dependency on each state (including where it was found) + res <- fromMaybe (f rest) (return <$> get (stackState s) key) + modifySTRef (dependencies s) $ S.insert $ dep key res + return res + +-- Like lookupStack but without adding dependencies +peekStack get def ctx key = do + top <- readSTRef $ cInput ctx + case get top key of + Just v -> return v + Nothing -> f (cStack ctx) + where + f [] = return def + f (s:rest) = + case get (stackState s) key of + Just v -> return v + Nothing -> f rest + +-- Check if the current context fulfills a StateDependency +fulfillsDependency ctx dep = + case dep of + DepGlobalValue name val -> (== (val, GlobalVar)) <$> peek ctx name + DepLocalValue name val -> (== (val, LocalVar)) <$> peek ctx name + DepPrefixValue name val -> (== (val, PrefixVar)) <$> peek ctx name + DepFunction name val -> (== val) <$> peekFunc ctx name + DepIsRecursive node val -> return $ val == any (\f -> entryPoint f == node) (cStack ctx) + -- _ -> error $ "Unknown dep " ++ show dep + where + peek = peekStack getVariableWithScope (unknownVariableValue, GlobalVar) + peekFunc = peekStack (\state name -> vmLookup name $ sFunctionTargets state) unknownFunctionValue + +-- Check if the current context fulfills all StateDependencies +fulfillsDependencies ctx deps = + f $ S.toList deps + where + f [] = return True + f (dep:rest) = do + res <- fulfillsDependency ctx dep + if res + then f rest + else return False + +-- Create a brand new Ctx given a Control Flow Graph (CFG) +newCtx g = do + c <- newSTRef 1 + input <- newSTRef undefined + output <- newSTRef undefined + node <- newSTRef undefined + cache <- newSTRef M.empty + invocations <- newSTRef M.empty + return $ Ctx { + cCounter = c, + cInput = input, + cOutput = output, + cNode = node, + cCache = cache, + cStack = [], + cInvocations = invocations, + cGraph = g + } + +-- The next incrementing version for VersionedMaps +nextVersion ctx = do + let ctr = cCounter ctx + n <- readSTRef ctr + writeSTRef ctr $! n+1 + return n + +-- Create a new StackEntry +newStackEntry ctx point = do + deps <- newSTRef S.empty + state <- readSTRef $ cOutput ctx + callsite <- readSTRef $ cNode ctx + return $ StackEntry { + entryPoint = point, + callSite = callsite, + dependencies = deps, + stackState = state + } + +-- Call a function with a new stack entry on the stack +withNewStackFrame ctx node f = do + newEntry <- newStackEntry ctx node + newInput <- newSTRef newInternalState + newOutput <- newSTRef newInternalState + newNode <- newSTRef node + let newCtx = ctx { + cInput = newInput, + cOutput = newOutput, + cNode = newNode, + cStack = newEntry : cStack ctx + } + x <- f newCtx + + {- + deps <- readSTRef $ dependencies newEntry + selfcheck <- fulfillsDependencies newCtx deps + unless selfcheck $ error $ pleaseReport $ "Unmet stack dependencies on " ++ show (node, deps) + -} + + return (x, newEntry) + +-- Check if invoking this function would be a recursive loop +-- (i.e. we already have the function on the stack) +wouldBeRecursive ctx node = f (cStack ctx) + where + f [] = return False + f (s:rest) = do + res <- + if entryPoint s == node + then return True + else f rest + modifySTRef (dependencies s) $ S.insert $ DepIsRecursive node res + return res + +-- The main DFA 'transfer' function, applying the effects of a node to the output state +transfer ctx label = + --traceShow ("Transferring", label) $ + case label of + CFStructuralNode -> return () + CFEntryPoint _ -> return () + CFImpliedExit -> return () + CFResolvedExit {} -> return () + + CFExecuteCommand cmd -> transferCommand ctx cmd + CFExecuteSubshell reason entry exit -> transferSubshell ctx reason entry exit + CFApplyEffects effects -> mapM_ (\(IdTagged _ f) -> transferEffect ctx f) effects + + CFUnresolvedExit -> patchOutputM ctx unreachableState + CFUnreachable -> patchOutputM ctx unreachableState + + -- TODO + CFSetBackgroundPid _ -> return () + CFSetExitCode _ -> return () + CFDropPrefixAssignments {} -> + modifySTRef (cOutput ctx) $ \c -> modified c { sPrefixValues = vmEmpty } +-- _ -> error $ "Unknown " ++ show label + + +-- Transfer the effects of a subshell invocation. This is similar to a function call +-- to allow easily discarding the effects (otherwise the InternalState would have +-- to represent subshell depth, while this way it can simply use the function stack). +transferSubshell ctx reason entry exit = do + let cout = cOutput ctx + initial <- readSTRef cout + runCached ctx entry (f entry exit) + -- Clear subshell changes. TODO: track this to warn about modifications. + writeSTRef cout initial + where + f entry exit ctx = do + (states, frame) <- withNewStackFrame ctx entry (flip dataflow $ entry) + let (_, res) = fromMaybe (error $ pleaseReport "Subshell has no exit") $ M.lookup exit states + deps <- readSTRef $ dependencies frame + registerFlowResult ctx entry states deps + return (deps, res) + +-- Transfer the effects of executing a command, i.e. the merged union of all possible function definitions. +transferCommand ctx Nothing = return () +transferCommand ctx (Just name) = do + targets <- readFunction ctx name + --traceShowM ("Transferring ",name,targets) + transferMultiple ctx $ map (flip transferFunctionValue) $ S.toList targets + +-- Transfer a set of function definitions and merge the output states. +transferMultiple ctx funcs = do +-- traceShowM ("Transferring set of ", length funcs) + original <- readSTRef out + branches <- mapM (apply ctx original) funcs + merged <- mergeStates ctx original branches + let patched = patchState original merged + writeSTRef out patched + where + out = cOutput ctx + apply ctx original f = do + writeSTRef out original + f ctx + readSTRef out + +-- Transfer the effects of a single function definition. +transferFunctionValue ctx funcVal = + case funcVal of + FunctionUnknown -> return () + FunctionDefinition name entry exit -> do + isRecursive <- wouldBeRecursive ctx entry + if isRecursive + then return () -- TODO: Find a better strategy for recursion + else runCached ctx entry (f name entry exit) + where + f name entry exit ctx = do + (states, frame) <- withNewStackFrame ctx entry (flip dataflow $ entry) + deps <- readSTRef $ dependencies frame + let res = + case M.lookup exit states of + Just (input, output) -> do + -- Discard local variables. TODO: track&retain variables declared local in previous scopes? + modified output { sLocalValues = vmEmpty } + Nothing -> do + -- e.g. f() { exit; } + unreachableState + registerFlowResult ctx entry states deps + return (deps, res) + + +-- Register/save the result of a dataflow of a function. +-- At the end, all the different values from different flows are merged together. +registerFlowResult ctx entry states deps = do + -- This function is called in the context of a CFExecuteCommand and not its invoked function, + -- so manually add the current node to the stack. + current <- readSTRef $ cNode ctx + let parents = map callSite $ cStack ctx + -- A unique path to this flow context. The specific value doesn't matter, as long as it's + -- unique per invocation of the function. This is required so that 'x=1; f; x=2; f' won't + -- overwrite each other. + let path = entry : current : parents + modifySTRef (cInvocations ctx) $ M.insert path (deps, states) + + +-- Look up a node in the cache and see if the dependencies of any entries are matched. +-- In that case, reuse the previous result instead of doing a new data flow. +runCached :: forall s. Ctx s -> Node -> (Ctx s -> ST s (S.Set StateDependency, InternalState)) -> ST s () +runCached ctx node f = do + cache <- getCache ctx node + case cache of + Just v -> do + -- traceShowM $ ("Running cached", node) + patchOutputM ctx v + Nothing -> do + -- traceShowM $ ("Cache failed", node) + (deps, diff) <- f ctx + modifySTRef (cCache ctx) (M.insertWith (\_ old -> (deps, diff):(take cacheEntries old)) node [(deps,diff)]) + -- traceShowM $ ("Recomputed cache for", node, deps) + patchOutputM ctx diff + +-- Get a cached version whose dependencies are currently fulfilled, if any. +getCache :: forall s. Ctx s -> Node -> ST s (Maybe InternalState) +getCache ctx node = do + cache <- readSTRef $ cCache ctx + -- traceShowM $ ("Cache for", node, "length", length $ M.findWithDefault [] node cache, M.lookup node cache) + f $ M.findWithDefault [] node cache + where + f [] = return Nothing + f ((deps, value):rest) = do + match <- fulfillsDependencies ctx deps + if match + then return $ Just value + else f rest + +-- Transfer a single CFEffect to the output state. +transferEffect ctx effect = + case effect of + CFReadVariable name -> do + void $ readVariable ctx name + CFWriteVariable name value -> do + val <- cfValueToVariableValue ctx value + writeVariable ctx name val + CFWriteGlobal name value -> do + val <- cfValueToVariableValue ctx value + writeGlobal ctx name val + CFWriteLocal name value -> do + val <- cfValueToVariableValue ctx value + writeLocal ctx name val + CFWritePrefix name value -> do + val <- cfValueToVariableValue ctx value + writePrefix ctx name val + CFUndefineVariable name -> undefineVariable ctx name + CFUndefineFunction name -> undefineFunction ctx name + CFUndefine name -> do + -- This should really just unset one or the other + undefineVariable ctx name + undefineFunction ctx name + CFDefineFunction name id entry exit -> + writeFunction ctx name $ FunctionDefinition name entry exit + + -- TODO + CFUndefineNameref name -> undefineVariable ctx name + CFHintArray name -> return () + CFHintDefined name -> return () + CFModifyProps {} -> return () +-- _ -> error $ "Unknown effect " ++ show effect + + + +-- Transfer the CFG's idea of a value into our VariableState +cfValueToVariableValue ctx val = + case val of + CFValueArray -> return unknownVariableValue -- TODO: Track array status + CFValueComputed _ parts -> foldM f emptyVariableValue parts + CFValueInteger -> return unknownIntegerValue + CFValueString -> return unknownVariableValue + CFValueUninitialized -> return emptyVariableValue +-- _ -> error $ "Unknown value: " ++ show val + where + f val part = do + next <- computeValue ctx part + return $ val `appendVariableValue` next + +-- A value can be computed from 0 or more parts, such as x="literal$y$z" +computeValue ctx part = + case part of + CFStringLiteral str -> return $ literalToVariableValue str + CFStringInteger -> return unknownIntegerValue + CFStringUnknown -> return unknownVariableValue + CFStringVariable name -> readVariable ctx name + +-- Append two VariableValues as if with z="$x$y" +appendVariableValue :: VariableValue -> VariableValue -> VariableValue +appendVariableValue a b = + VariableValue { + literalValue = liftM2 (++) (literalValue a) (literalValue b), + spaceStatus = appendSpaceStatus (spaceStatus a) (spaceStatus b) + } + +appendSpaceStatus a b = + case (a,b) of + (SpaceStatusEmpty, _) -> b + (_, SpaceStatusEmpty) -> a + (SpaceStatusClean, SpaceStatusClean) -> a + _ ->SpaceStatusDirty + +unknownIntegerValue = VariableValue { + literalValue = Nothing, + spaceStatus = SpaceStatusClean +} + +literalToVariableValue str = VariableValue { + literalValue = Just str, + spaceStatus = literalToSpaceStatus str +} + +withoutChanges ctx f = do + let inp = cInput ctx + let out = cOutput ctx + prevInput <- readSTRef inp + prevOutput <- readSTRef out + res <- f + writeSTRef inp prevInput + writeSTRef out prevOutput + return res + +-- Get the SpaceStatus for a literal string, i.e. if it needs quoting +literalToSpaceStatus str = + case str of + "" -> SpaceStatusEmpty + _ | all (`notElem` " \t\n*?[") str -> SpaceStatusClean + _ -> SpaceStatusDirty + +type StateMap = M.Map Node (InternalState, InternalState) + +-- Classic, iterative Data Flow Analysis. See Wikipedia for a description of the process. +dataflow :: forall s. Ctx s -> Node -> ST s StateMap +dataflow ctx entry = do + pending <- newSTRef $ S.singleton entry + states <- newSTRef $ M.empty + -- Should probably be done via a stack frame instead + withoutChanges ctx $ + f iterationCount pending states + readSTRef states + where + graph = cGraph ctx + f 0 _ _ = error $ pleaseReport "DFA did not reach fix point" + f n pending states = do + ps <- readSTRef pending + if S.null ps + then return () + else do + let (next, rest) = S.deleteFindMin ps + nexts <- process states next + writeSTRef pending $ foldl (flip S.insert) rest nexts + f (n-1) pending states + + process states node = do + stateMap <- readSTRef states + let inputs = filter (\c -> sIsReachable c /= Just False) $ mapMaybe (\c -> fmap snd $ M.lookup c stateMap) incoming + input <- + case incoming of + [] -> return newInternalState + _ -> + case inputs of + [] -> return unreachableState + (x:rest) -> foldM (mergeState ctx) x rest + writeSTRef (cInput ctx) $ input + writeSTRef (cOutput ctx) $ input + writeSTRef (cNode ctx) $ node + transfer ctx label + newOutput <- readSTRef $ cOutput ctx + result <- + if is2plus outgoing + then + -- Version the state because we split and will probably merge later + versionState ctx newOutput + else return newOutput + writeSTRef states $ M.insert node (input, result) stateMap + case M.lookup node stateMap of + Nothing -> return outgoing + Just (oldInput, oldOutput) -> + if oldOutput == result + then return [] + else return outgoing + where + (incomingL, _, label, outgoingL) = context graph $ node + incoming = map snd $ filter isRegular $ incomingL + outgoing = map snd outgoingL + isRegular = ((== CFEFlow) . fst) + +runRoot ctx entry exit = do + let env = createEnvironmentState + writeSTRef (cInput ctx) $ env + writeSTRef (cOutput ctx) $ env + writeSTRef (cNode ctx) $ entry + (states, frame) <- withNewStackFrame ctx entry $ \c -> dataflow c entry + deps <- readSTRef $ dependencies frame + registerFlowResult ctx entry states deps + -- Return the final state, used to invoke functions that were declared but not invoked + return $ snd $ fromMaybe (error $ pleaseReport "Missing exit state") $ M.lookup exit states + + +analyzeControlFlow :: CFGParameters -> Token -> CFGAnalysis +analyzeControlFlow params t = + let + cfg = buildGraph params t + (entry, exit) = M.findWithDefault (error $ pleaseReport "Missing root") (getId t) (cfIdToNode cfg) + in + runST $ f cfg entry exit + where + f cfg entry exit = do + ctx <- newCtx $ cfGraph cfg + -- Do a dataflow analysis starting on the root node + exitState <- runRoot ctx entry exit + + -- All nodes we've touched + invocations <- readSTRef $ cInvocations ctx + let invokedNodes = M.fromDistinctAscList $ map (\c -> (c, ())) $ S.toList $ M.keysSet $ groupByNode $ M.map snd invocations + + -- Invoke all functions that were declared but not invoked + -- This is so that we still get warnings for dead code + -- (it's probably not actually dead, just used by a script that sources ours) + let declaredFunctions = getFunctionTargets exitState + let uninvoked = M.difference declaredFunctions invokedNodes + analyzeStragglers ctx exitState uninvoked + + -- Now round up all the states from all data flows + -- (FIXME: this excludes functions that were defined in straggling functions) + invocations <- readSTRef $ cInvocations ctx + invokedStates <- flattenByNode ctx $ groupByNode $ M.map addDeps invocations + + -- Fill in the map with unreachable states for anything we didn't get to + let baseStates = M.fromDistinctAscList $ map (\c -> (c, (unreachableState, unreachableState))) $ uncurry enumFromTo $ nodeRange $ cfGraph cfg + let allStates = M.unionWith (flip const) baseStates invokedStates + + -- Convert to external states + let nodeToData = M.map (\(a,b) -> (internalToExternal a, internalToExternal b)) allStates + + return $ nodeToData `deepseq` CFGAnalysis { + graph = cfGraph cfg, + tokenToNode = cfIdToNode cfg, + nodeToData = nodeToData + } + + + -- Include the dependencies in the state of each function, e.g. if it depends on `x=foo` then add that. + addDeps :: (S.Set StateDependency, M.Map Node (InternalState, InternalState)) -> M.Map Node (InternalState, InternalState) + addDeps (deps, m) = let base = depsToState deps in M.map (\(a,b) -> (base `patchState` a, base `patchState` b)) m + + -- Collect all the states that each node has resulted in. + groupByNode :: forall k v. M.Map k (M.Map Node v) -> M.Map Node [v] + groupByNode pathMap = M.fromListWith (++) $ map (\(k,v) -> (k,[v])) $ concatMap M.toList $ M.elems pathMap + + -- Merge all the pre/post states for each node. This would have been a foldM if Map had one. + flattenByNode ctx m = M.fromDistinctAscList <$> (mapM (mergePair ctx) $ M.toList m) + + mergeAllStates ctx pairs = + let + (pres, posts) = unzip pairs + in do + pre <- mergeStates ctx (error $ pleaseReport "Null node states") pres + post <- mergeStates ctx (error $ pleaseReport "Null node states") posts + return (pre, post) + + mergePair ctx (node, list) = do + merged <- mergeAllStates ctx list + return (node, merged) + + -- Get the all the functions defined in an InternalState + getFunctionTargets :: InternalState -> M.Map Node FunctionDefinition + getFunctionTargets state = + let + declaredFuncs = S.unions $ mapStorage $ sFunctionTargets state + getFunc d = + case d of + FunctionDefinition _ entry _ -> Just (entry, d) + _ -> Nothing + funcs = mapMaybe getFunc $ S.toList declaredFuncs + in + M.fromList funcs + + +analyzeStragglers ctx state stragglers = do + mapM_ f $ M.elems stragglers + where + f def@(FunctionDefinition name entry exit) = do + writeSTRef (cInput ctx) state + writeSTRef (cOutput ctx) state + writeSTRef (cNode ctx) entry + transferFunctionValue ctx def + + +return [] +runTests = $quickCheckAll diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index e65dc68..cac06bc 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -30,6 +30,7 @@ import ShellCheck.AnalyzerLib import ShellCheck.Data import ShellCheck.Interface import ShellCheck.Parser +import ShellCheck.Prelude import ShellCheck.Regex import Control.Monad diff --git a/src/ShellCheck/Checks/ControlFlow.hs b/src/ShellCheck/Checks/ControlFlow.hs new file mode 100644 index 0000000..9b7635e --- /dev/null +++ b/src/ShellCheck/Checks/ControlFlow.hs @@ -0,0 +1,101 @@ +{- + Copyright 2022 Vidar Holen + + This file is part of ShellCheck. + https://www.shellcheck.net + + ShellCheck is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + ShellCheck is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +-} +{-# LANGUAGE TemplateHaskell #-} + +-- Checks that run on the Control Flow Graph (as opposed to the AST) +-- This is scaffolding for a work in progress. + +module ShellCheck.Checks.ControlFlow (checker, optionalChecks, ShellCheck.Checks.ControlFlow.runTests) where + +import ShellCheck.AST +import ShellCheck.ASTLib +import ShellCheck.CFG hiding (cfgAnalysis) +import ShellCheck.CFGAnalysis +import ShellCheck.AnalyzerLib +import ShellCheck.Data +import ShellCheck.Interface + +import Control.Monad +import Control.Monad.Reader +import Data.Graph.Inductive.Graph +import qualified Data.Map as M +import qualified Data.Set as S +import Data.List +import Data.Maybe + +import Test.QuickCheck.All (forAllProperties) +import Test.QuickCheck.Test (quickCheckWithResult, stdArgs, maxSuccess) + + +optionalChecks :: [CheckDescription] +optionalChecks = [] + +-- A check that runs on the entire graph +type ControlFlowCheck = Analysis +-- A check invoked once per node, with its (pre,post) data +type ControlFlowNodeCheck = LNode CFNode -> (ProgramState, ProgramState) -> Analysis +-- A check invoked once per effect, with its node's (pre,post) data +type ControlFlowEffectCheck = IdTagged CFEffect -> Node -> (ProgramState, ProgramState) -> Analysis + + +checker :: AnalysisSpec -> Parameters -> Checker +checker spec params = Checker { + perScript = const $ sequence_ controlFlowChecks, + perToken = const $ return () +} + +controlFlowChecks :: [ControlFlowCheck] +controlFlowChecks = [ + runNodeChecks controlFlowNodeChecks + ] + +controlFlowNodeChecks :: [ControlFlowNodeCheck] +controlFlowNodeChecks = [ + runEffectChecks controlFlowEffectChecks + ] + +controlFlowEffectChecks :: [ControlFlowEffectCheck] +controlFlowEffectChecks = [ + ] + +runNodeChecks :: [ControlFlowNodeCheck] -> ControlFlowCheck +runNodeChecks perNode = do + cfg <- asks cfgAnalysis + runOnAll cfg + where + getData datas n@(node, label) = do + (pre, post) <- M.lookup node datas + return (n, (pre, post)) + + runOn :: (LNode CFNode, (ProgramState, ProgramState)) -> Analysis + runOn (node, prepost) = mapM_ (\c -> c node prepost) perNode + runOnAll cfg = mapM_ runOn $ mapMaybe (getData $ nodeToData cfg) $ labNodes (graph cfg) + +runEffectChecks :: [ControlFlowEffectCheck] -> ControlFlowNodeCheck +runEffectChecks list = checkNode + where + checkNode (node, label) prepost = + case label of + CFApplyEffects effects -> mapM_ (\effect -> mapM_ (\c -> c effect node prepost) list) effects + _ -> return () + + +return [] +runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 22a6a5f..9ad17f5 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -25,6 +25,7 @@ import ShellCheck.AST import ShellCheck.ASTLib import ShellCheck.AnalyzerLib import ShellCheck.Interface +import ShellCheck.Prelude import ShellCheck.Regex import Control.Monad diff --git a/src/ShellCheck/Data.hs b/src/ShellCheck/Data.hs index e22b424..fb82ca8 100644 --- a/src/ShellCheck/Data.hs +++ b/src/ShellCheck/Data.hs @@ -2,9 +2,27 @@ module ShellCheck.Data where import ShellCheck.Interface import Data.Version (showVersion) -import Paths_ShellCheck (version) -shellcheckVersion = showVersion version -- VERSIONSTRING + +{- +If you are here because you saw an error about Paths_ShellCheck in this file, +simply comment out the import below and define the version as a constant string. + +Instead of: + + import Paths_ShellCheck (version) + shellcheckVersion = showVersion version + +Use: + + -- import Paths_ShellCheck (version) + shellcheckVersion = "kludge" + +-} + +import Paths_ShellCheck (version) +shellcheckVersion = showVersion version -- VERSIONSTRING + internalVariables = [ -- Generic @@ -43,9 +61,12 @@ internalVariables = [ "flags_error", "flags_return" ] -specialVariablesWithoutSpaces = [ - "$", "-", "?", "!", "#" +specialIntegerVariables = [ + "$", "?", "!", "#" ] + +specialVariablesWithoutSpaces = "-" : specialIntegerVariables + variablesWithoutSpaces = specialVariablesWithoutSpaces ++ [ "BASHPID", "BASH_ARGC", "BASH_LINENO", "BASH_SUBSHELL", "EUID", "LINENO", "OPTIND", "PPID", "RANDOM", "SECONDS", "SHELLOPTS", "SHLVL", "UID", diff --git a/src/ShellCheck/Debug.hs b/src/ShellCheck/Debug.hs new file mode 100644 index 0000000..c991308 --- /dev/null +++ b/src/ShellCheck/Debug.hs @@ -0,0 +1,313 @@ +{- + +This file contains useful functions for debugging and developing ShellCheck. + +To invoke them interactively, run: + + cabal repl + +At the ghci prompt, enter: + + :load ShellCheck.Debug + +You can now invoke the functions. Here are some examples: + + shellcheckString "echo $1" + stringToAst "(( x+1 ))" + stringToCfg "if foo; then bar; else baz; fi" + writeFile "/tmp/test.dot" $ stringToCfgViz "while foo; do bar; done" + +The latter file can be rendered to png with GraphViz: + + dot -Tpng /tmp/test.dot > /tmp/test.png + +To run all unit tests in a module: + + ShellCheck.Parser.runTests + ShellCheck.Analytics.runTests + +To run a specific test: + + :load ShellCheck.Analytics + prop_checkUuoc3 + +If you make code changes, reload in seconds at any time with: + + :r + +=========================================================================== + +Crash course in printf debugging in Haskell: + + import Debug.Trace + + greet 0 = return () + -- Print when a function is invoked + greet n | trace ("calling greet " ++ show n) False = undefined + greet n = do + putStrLn "Enter name" + name <- getLine + -- Print at some point in any monadic function + traceM $ "user entered " ++ name + putStrLn $ "Hello " ++ name + -- Print a value before passing it on + greet $ traceShowId (n - 1) + + +=========================================================================== + +If you want to invoke `ghci` directly, such as on `shellcheck.hs`, to +debug all of ShellCheck including I/O, you may see an error like this: + + src/ShellCheck/Data.hs:5:1: error: + Could not load module ‘Paths_ShellCheck’ + it is a hidden module in the package ‘ShellCheck-0.8.0’ + +This can easily be circumvented by running `./setgitversion` or manually +editing src/ShellCheck/Data.hs to replace the auto-deduced version number +with a constant string as indicated. + +Afterwards, you can run the ShellCheck tool, as if from the shell, with: + + $ ghci shellcheck.hs + ghci> runMain ["-x", "file.sh"] + +-} + +module ShellCheck.Debug () where + +import ShellCheck.Analyzer +import ShellCheck.AST +import ShellCheck.CFG +import ShellCheck.Checker +import ShellCheck.CFGAnalysis as CF +import ShellCheck.Interface +import ShellCheck.Parser +import ShellCheck.Prelude + +import Control.Monad +import Control.Monad.Identity +import Control.Monad.RWS +import Control.Monad.Writer +import Data.Graph.Inductive.Graph as G +import Data.List +import Data.Maybe +import qualified Data.Map as M +import qualified Data.Set as S + + +-- Run all of ShellCheck (minus output formatters) +shellcheckString :: String -> CheckResult +shellcheckString scriptString = + runIdentity $ checkScript dummySystemInterface checkSpec + where + checkSpec :: CheckSpec + checkSpec = emptyCheckSpec { + csScript = scriptString + } + +dummySystemInterface :: SystemInterface Identity +dummySystemInterface = mockedSystemInterface [ + -- A tiny, fake filesystem for sourced files + ("lib/mylib1.sh", "foo=$(cat $1 | wc -l)"), + ("lib/mylib2.sh", "bar=42") + ] + +-- Parameters used when generating Control Flow Graphs +cfgParams :: CFGParameters +cfgParams = CFGParameters { + cfLastpipe = False, + cfPipefail = False +} + +-- An example script to play with +exampleScript :: String +exampleScript = unlines [ + "#!/bin/sh", + "count=0", + "for file in *", + "do", + " (( count++ ))", + "done", + "echo $count" + ] + +-- Parse the script string into ShellCheck's ParseResult +parseScriptString :: String -> ParseResult +parseScriptString scriptString = + runIdentity $ parseScript dummySystemInterface parseSpec + where + parseSpec :: ParseSpec + parseSpec = newParseSpec { + psFilename = "myscript", + psScript = scriptString + } + + +-- Parse the script string into an Abstract Syntax Tree +stringToAst :: String -> Token +stringToAst scriptString = + case maybeRoot of + Just root -> root + Nothing -> error $ "Script failed to parse: " ++ show parserWarnings + where + parseResult :: ParseResult + parseResult = parseScriptString scriptString + + maybeRoot :: Maybe Token + maybeRoot = prRoot parseResult + + parserWarnings :: [PositionedComment] + parserWarnings = prComments parseResult + + +astToCfgResult :: Token -> CFGResult +astToCfgResult = buildGraph cfgParams + +astToDfa :: Token -> CFGAnalysis +astToDfa = analyzeControlFlow cfgParams + +astToCfg :: Token -> CFGraph +astToCfg = cfGraph . astToCfgResult + +stringToCfg :: String -> CFGraph +stringToCfg = astToCfg . stringToAst + +stringToDfa :: String -> CFGAnalysis +stringToDfa = astToDfa . stringToAst + +cfgToGraphViz :: CFGraph -> String +cfgToGraphViz = cfgToGraphVizWith show + +stringToCfgViz :: String -> String +stringToCfgViz = cfgToGraphViz . stringToCfg + +stringToDfaViz :: String -> String +stringToDfaViz = dfaToGraphViz . stringToDfa + +-- Dump a Control Flow Graph as GraphViz with extended information +stringToDetailedCfgViz :: String -> String +stringToDetailedCfgViz scriptString = cfgToGraphVizWith nodeLabel graph + where + ast :: Token + ast = stringToAst scriptString + + cfgResult :: CFGResult + cfgResult = astToCfgResult ast + + graph :: CFGraph + graph = cfGraph cfgResult + + idToToken :: M.Map Id Token + idToToken = M.fromList $ execWriter $ doAnalysis (\c -> tell [(getId c, c)]) ast + + idToNode :: M.Map Id (Node, Node) + idToNode = cfIdToNode cfgResult + + nodeToStartIds :: M.Map Node (S.Set Id) + nodeToStartIds = + M.fromListWith S.union $ + map (\(id, (start, _)) -> (start, S.singleton id)) $ + M.toList idToNode + + nodeToEndIds :: M.Map Node (S.Set Id) + nodeToEndIds = + M.fromListWith S.union $ + map (\(id, (_, end)) -> (end, S.singleton id)) $ + M.toList idToNode + + formatId :: Id -> String + formatId id = fromMaybe ("Unknown " ++ show id) $ do + (OuterToken _ token) <- M.lookup id idToToken + firstWord <- words (show token) !!! 0 + -- Strip off "Inner_" + (_ : tokenName) <- return $ dropWhile (/= '_') firstWord + return $ tokenName ++ " " ++ show id + + formatGroup :: S.Set Id -> String + formatGroup set = intercalate ", " $ map formatId $ S.toList set + + nodeLabel (node, label) = unlines [ + show node ++ ". " ++ show label, + "Begin: " ++ formatGroup (M.findWithDefault S.empty node nodeToStartIds), + "End: " ++ formatGroup (M.findWithDefault S.empty node nodeToEndIds) + ] + + +-- Dump a Control Flow Graph with Data Flow Analysis as GraphViz +dfaToGraphViz :: CF.CFGAnalysis -> String +dfaToGraphViz analysis = cfgToGraphVizWith label $ CF.graph analysis + where + label (node, label) = + let + desc = show node ++ ". " ++ show label + in + fromMaybe ("No DFA available\n\n" ++ desc) $ do + (pre, post) <- M.lookup node $ CF.nodeToData analysis + return $ unlines [ + "Precondition: " ++ show pre, + "", + desc, + "", + "Postcondition: " ++ show post + ] + + +-- Dump an Control Flow Graph to GraphViz with a given node formatter +cfgToGraphVizWith :: (LNode CFNode -> String) -> CFGraph -> String +cfgToGraphVizWith nodeLabel graph = concat [ + "digraph {\n", + concatMap dumpNode (labNodes graph), + concatMap dumpLink (labEdges graph), + tagVizEntries graph, + "}\n" + ] + where + dumpNode l@(node, label) = show node ++ " [label=" ++ quoteViz (nodeLabel l) ++ "]\n" + dumpLink (from, to, typ) = show from ++ " -> " ++ show to ++ " [style=" ++ quoteViz (edgeStyle typ) ++ "]\n" + edgeStyle CFEFlow = "solid" + edgeStyle CFEExit = "bold" + edgeStyle CFEFalseFlow = "dotted" + +quoteViz str = "\"" ++ escapeViz str ++ "\"" +escapeViz [] = [] +escapeViz (c:rest) = + case c of + '\"' -> '\\' : '\"' : escapeViz rest + '\n' -> '\\' : 'l' : escapeViz rest + '\\' -> '\\' : '\\' : escapeViz rest + _ -> c : escapeViz rest + + +-- Dump an Abstract Syntax Tree (or branch thereof) to GraphViz format +astToGraphViz :: Token -> String +astToGraphViz token = concat [ + "digraph {\n", + formatTree token, + "}\n" + ] + where + formatTree :: Token -> String + formatTree t = snd $ execRWS (doStackAnalysis push pop t) () [] + + push :: Token -> RWS () String [Int] () + push (OuterToken (Id n) inner) = do + stack <- get + put (n : stack) + case stack of + [] -> return () + (top:_) -> tell $ show top ++ " -> " ++ show n ++ "\n" + tell $ show n ++ " [label=" ++ quoteViz (show n ++ ": " ++ take 32 (show inner)) ++ "]\n" + + pop :: Token -> RWS () String [Int] () + pop _ = modify tail + + +-- For each entry point, set the rank so that they'll align in the graph +tagVizEntries :: CFGraph -> String +tagVizEntries graph = "{ rank=same " ++ rank ++ " }" + where + entries = mapMaybe find $ labNodes graph + find (node, CFEntryPoint name) = return (node, name) + find _ = Nothing + rank = unwords $ map (\(c, _) -> show c) entries diff --git a/src/ShellCheck/Fixer.hs b/src/ShellCheck/Fixer.hs index 1409b24..2376842 100644 --- a/src/ShellCheck/Fixer.hs +++ b/src/ShellCheck/Fixer.hs @@ -22,6 +22,7 @@ module ShellCheck.Fixer (applyFix, removeTabStops, mapPositions, Ranged(..), runTests) where import ShellCheck.Interface +import ShellCheck.Prelude import Control.Monad.State import Data.Array import Data.List @@ -228,7 +229,7 @@ applyReplacement2 rep string = do let (l1, l2) = tmap posLine originalPos in when (l1 /= 1 || l2 /= 1) $ - error "ShellCheck internal error, please report: bad cross-line fix" + error $ pleaseReport "bad cross-line fix" let replacer = repString rep let shift = (length replacer) - (oldEnd - oldStart) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 3958406..9f9241c 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -27,6 +27,7 @@ import ShellCheck.AST import ShellCheck.ASTLib hiding (runTests) import ShellCheck.Data import ShellCheck.Interface +import ShellCheck.Prelude import Control.Applicative ((<*), (*>)) import Control.Monad @@ -210,7 +211,7 @@ getNextIdSpanningTokenList list = -- Get the span covered by an id getSpanForId :: Monad m => Id -> SCParser m (SourcePos, SourcePos) getSpanForId id = - Map.findWithDefault (error "Internal error: no position for id. Please report!") id <$> + Map.findWithDefault (error $ pleaseReport "no parser span for id") id <$> getMap -- Create a new id with the same span as an existing one @@ -1918,7 +1919,7 @@ readPendingHereDocs = do -- The end token is just a prefix skipLine | hasTrailer -> - error "ShellCheck bug, please report (here doc trailer)." + error $ pleaseReport "unexpected heredoc trailer" -- The following cases assume no trailing text: | dashed == Undashed && (not $ null leadingSpace) -> do diff --git a/src/ShellCheck/Prelude.hs b/src/ShellCheck/Prelude.hs new file mode 100644 index 0000000..7e9011b --- /dev/null +++ b/src/ShellCheck/Prelude.hs @@ -0,0 +1,48 @@ +{- + Copyright 2022 Vidar Holen + + This file is part of ShellCheck. + https://www.shellcheck.net + + ShellCheck is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + ShellCheck is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +-} + +-- Generic basic utility functions +module ShellCheck.Prelude where + +-- Get element 0 or a default. Like `head` but safe. +headOrDefault _ (a:_) = a +headOrDefault def _ = def + +-- Get the last element or a default. Like `last` but safe. +lastOrDefault def [] = def +lastOrDefault _ list = last list + +--- Get element n of a list, or Nothing. Like `!!` but safe. +(!!!) list i = + case drop i list of + [] -> Nothing + (r:_) -> Just r + + +-- Like mconcat but for Semigroups +sconcat1 :: (Semigroup t) => [t] -> t +sconcat1 [x] = x +sconcat1 (x:xs) = x <> sconcat1 xs + +sconcatOrDefault def [] = def +sconcatOrDefault _ list = sconcat1 list + +-- For more actionable "impossible" errors +pleaseReport str = "ShellCheck internal error, please report: " ++ str diff --git a/test/shellcheck.hs b/test/shellcheck.hs index e463403..1a272af 100644 --- a/test/shellcheck.hs +++ b/test/shellcheck.hs @@ -5,8 +5,11 @@ import System.Exit import qualified ShellCheck.Analytics import qualified ShellCheck.AnalyzerLib import qualified ShellCheck.ASTLib +import qualified ShellCheck.CFG +import qualified ShellCheck.CFGAnalysis import qualified ShellCheck.Checker import qualified ShellCheck.Checks.Commands +import qualified ShellCheck.Checks.ControlFlow import qualified ShellCheck.Checks.Custom import qualified ShellCheck.Checks.ShellSupport import qualified ShellCheck.Fixer @@ -19,8 +22,11 @@ main = do ShellCheck.Analytics.runTests ,ShellCheck.AnalyzerLib.runTests ,ShellCheck.ASTLib.runTests + ,ShellCheck.CFG.runTests + ,ShellCheck.CFGAnalysis.runTests ,ShellCheck.Checker.runTests ,ShellCheck.Checks.Commands.runTests + ,ShellCheck.Checks.ControlFlow.runTests ,ShellCheck.Checks.Custom.runTests ,ShellCheck.Checks.ShellSupport.runTests ,ShellCheck.Fixer.runTests From 642ad8612597b28af4026bde289985583fddb69d Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 19 Jul 2022 14:33:00 -0700 Subject: [PATCH 239/459] Add SC2317 warning about unreachable commands --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 32 ++++++++++++++++++++++++-------- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4763ddd..6e1dba3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## Git ### Added - SC2316: Warn about 'local readonly foo' and similar (thanks, patrickxia!) +- SC2317: Warn about unreachable commands ### Fixed diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index bf5d179..e6c0160 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -202,6 +202,7 @@ nodeChecks = [ ,checkCommandWithTrailingSymbol ,checkUnquotedParameterExpansionPattern ,checkBatsTestDoesNotUseNegation + ,checkCommandIsUnreachable ] optionalChecks = map fst optionalTreeChecks @@ -4129,13 +4130,6 @@ checkAliasUsedInSameParsingUnit params root = checkUnit :: [Token] -> Writer [TokenComment] () checkUnit unit = evalStateT (mapM_ (doAnalysis findCommands) unit) (Map.empty) - isSourced t = - let - f (T_SourceCommand {}) = True - f _ = False - in - any f $ getPath (parentMap params) t - findCommands :: Token -> StateT (Map.Map String Token) (Writer [TokenComment]) () findCommands t = case t of T_SimpleCommand _ _ (cmd:args) -> @@ -4146,7 +4140,7 @@ checkAliasUsedInSameParsingUnit params root = cmd <- gets (Map.lookup name) case cmd of Just alias -> - unless (isSourced t || shouldIgnoreCode params 2262 alias) $ do + unless (isSourced params t || shouldIgnoreCode params 2262 alias) $ do warn (getId alias) 2262 "This alias can't be defined and used in the same parsing unit. Use a function instead." info (getId t) 2263 "Since they're in the same parsing unit, this command will not refer to the previously mentioned alias." _ -> return () @@ -4157,6 +4151,14 @@ checkAliasUsedInSameParsingUnit params root = when (isVariableName name && not (null value)) $ modify (Map.insertWith (\new old -> old) name arg) +isSourced params t = + let + f (T_SourceCommand {}) = True + f _ = False + in + any f $ getPath (parentMap params) t + + -- Like groupBy, but compares pairs of adjacent elements, rather than against the first of the span prop_groupByLink1 = groupByLink (\a b -> a+1 == b) [1,2,3,2,3,7,8,9] == [[1,2,3], [2,3], [7,8,9]] prop_groupByLink2 = groupByLink (==) ([] :: [()]) == [] @@ -4910,5 +4912,19 @@ checkBatsTestDoesNotUseNegation params t = x:rest -> isLastOf t rest [] -> False + +prop_checkCommandIsUnreachable1 = verify checkCommandIsUnreachable "foo; bar; exit; baz" +prop_checkCommandIsUnreachable2 = verify checkCommandIsUnreachable "die() { exit; }; foo; bar; die; baz" +prop_checkCommandIsUnreachable3 = verifyNot checkCommandIsUnreachable "foo; bar || exit; baz" +checkCommandIsUnreachable params t = + case t of + T_Pipeline {} -> sequence_ $ do + state <- CF.getIncomingState (cfgAnalysis params) id + guard . not $ CF.stateIsReachable state + guard . not $ isSourced params t + return $ info id 2317 "Command appears to be unreachable. Check usage (or ignore if invoked indirectly)." + _ -> return () + where id = getId t + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) From da4885a71d08c76c6ff5d01012d135d54fa68f4e Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 19 Jul 2022 14:33:00 -0700 Subject: [PATCH 240/459] Use DFA for SC2086 --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 272 ++++++++++++++---------------------- 2 files changed, 104 insertions(+), 169 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e1dba3..66d5369 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - SC2317: Warn about unreachable commands ### Fixed +- SC2086: Now uses DFA to make more accurate predictions about values ### Changed - ShellCheck now has a Data Flow Analysis engine to make smarter decisions diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index e6c0160..0062879 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -54,7 +54,6 @@ treeChecks :: [Parameters -> Token -> [TokenComment]] treeChecks = [ nodeChecksToTreeCheck nodeChecks ,subshellAssignmentCheck - ,checkSpacefulness ,checkQuotesInLiterals ,checkShebangParameters ,checkFunctionsUsedExternally @@ -203,6 +202,7 @@ nodeChecks = [ ,checkUnquotedParameterExpansionPattern ,checkBatsTestDoesNotUseNegation ,checkCommandIsUnreachable + ,checkSpacefulnessCfg ] optionalChecks = map fst optionalTreeChecks @@ -221,7 +221,7 @@ optionalTreeChecks = [ cdDescription = "Suggest quoting variables without metacharacters", cdPositive = "var=hello; echo $var", cdNegative = "var=hello; echo \"$var\"" - }, checkVerboseSpacefulness) + }, nodeChecksToTreeCheck [checkVerboseSpacefulnessCfg]) ,(newCheckDescription { cdName = "avoid-nullary-conditions", @@ -2009,109 +2009,6 @@ doVariableFlowAnalysis readFunc writeFunc empty flow = evalState ( writeFunc base token name values doFlow _ = return [] ----- Check whether variables could have spaces/globs -prop_checkSpacefulness1 = verifyTree checkSpacefulness "a='cow moo'; echo $a" -prop_checkSpacefulness2 = verifyNotTree checkSpacefulness "a='cow moo'; [[ $a ]]" -prop_checkSpacefulness3 = verifyNotTree checkSpacefulness "a='cow*.mp3'; echo \"$a\"" -prop_checkSpacefulness4 = verifyTree checkSpacefulness "for f in *.mp3; do echo $f; done" -prop_checkSpacefulness4a= verifyNotTree checkSpacefulness "foo=3; foo=$(echo $foo)" -prop_checkSpacefulness5 = verifyTree checkSpacefulness "a='*'; b=$a; c=lol${b//foo/bar}; echo $c" -prop_checkSpacefulness6 = verifyTree checkSpacefulness "a=foo$(lol); echo $a" -prop_checkSpacefulness7 = verifyTree checkSpacefulness "a=foo\\ bar; rm $a" -prop_checkSpacefulness8 = verifyNotTree checkSpacefulness "a=foo\\ bar; a=foo; rm $a" -prop_checkSpacefulness10= verifyTree checkSpacefulness "rm $1" -prop_checkSpacefulness11= verifyTree checkSpacefulness "rm ${10//foo/bar}" -prop_checkSpacefulness12= verifyNotTree checkSpacefulness "(( $1 + 3 ))" -prop_checkSpacefulness13= verifyNotTree checkSpacefulness "if [[ $2 -gt 14 ]]; then true; fi" -prop_checkSpacefulness14= verifyNotTree checkSpacefulness "foo=$3 env" -prop_checkSpacefulness15= verifyNotTree checkSpacefulness "local foo=$1" -prop_checkSpacefulness16= verifyNotTree checkSpacefulness "declare foo=$1" -prop_checkSpacefulness17= verifyTree checkSpacefulness "echo foo=$1" -prop_checkSpacefulness18= verifyNotTree checkSpacefulness "$1 --flags" -prop_checkSpacefulness19= verifyTree checkSpacefulness "echo $PWD" -prop_checkSpacefulness20= verifyNotTree checkSpacefulness "n+='foo bar'" -prop_checkSpacefulness21= verifyNotTree checkSpacefulness "select foo in $bar; do true; done" -prop_checkSpacefulness22= verifyNotTree checkSpacefulness "echo $\"$1\"" -prop_checkSpacefulness23= verifyNotTree checkSpacefulness "a=(1); echo ${a[@]}" -prop_checkSpacefulness24= verifyTree checkSpacefulness "a='a b'; cat <<< $a" -prop_checkSpacefulness25= verifyTree checkSpacefulness "a='s/[0-9]//g'; sed $a" -prop_checkSpacefulness26= verifyTree checkSpacefulness "a='foo bar'; echo {1,2,$a}" -prop_checkSpacefulness27= verifyNotTree checkSpacefulness "echo ${a:+'foo'}" -prop_checkSpacefulness28= verifyNotTree checkSpacefulness "exec {n}>&1; echo $n" -prop_checkSpacefulness29= verifyNotTree checkSpacefulness "n=$(stuff); exec {n}>&-;" -prop_checkSpacefulness30= verifyTree checkSpacefulness "file='foo bar'; echo foo > $file;" -prop_checkSpacefulness31= verifyNotTree checkSpacefulness "echo \"`echo \\\"$1\\\"`\"" -prop_checkSpacefulness32= verifyNotTree checkSpacefulness "var=$1; [ -v var ]" -prop_checkSpacefulness33= verifyTree checkSpacefulness "for file; do echo $file; done" -prop_checkSpacefulness34= verifyTree checkSpacefulness "declare foo$n=$1" -prop_checkSpacefulness35= verifyNotTree checkSpacefulness "echo ${1+\"$1\"}" -prop_checkSpacefulness36= verifyNotTree checkSpacefulness "arg=$#; echo $arg" -prop_checkSpacefulness37= verifyNotTree checkSpacefulness "@test 'status' {\n [ $status -eq 0 ]\n}" -prop_checkSpacefulness37v = verifyTree checkVerboseSpacefulness "@test 'status' {\n [ $status -eq 0 ]\n}" -prop_checkSpacefulness38= verifyTree checkSpacefulness "a=; echo $a" -prop_checkSpacefulness39= verifyNotTree checkSpacefulness "a=''\"\"''; b=x$a; echo $b" -prop_checkSpacefulness40= verifyNotTree checkSpacefulness "a=$((x+1)); echo $a" -prop_checkSpacefulness41= verifyNotTree checkSpacefulness "exec $1 --flags" -prop_checkSpacefulness42= verifyNotTree checkSpacefulness "run $1 --flags" -prop_checkSpacefulness43= verifyNotTree checkSpacefulness "$foo=42" -prop_checkSpacefulness44= verifyTree checkSpacefulness "#!/bin/sh\nexport var=$value" -prop_checkSpacefulness45= verifyNotTree checkSpacefulness "wait -zzx -p foo; echo $foo" -prop_checkSpacefulness46= verifyNotTree checkSpacefulness "x=0; (( x += 1 )); echo $x" -prop_checkSpacefulness47= verifyNotTree checkSpacefulness "x=0; (( x-- )); echo $x" -prop_checkSpacefulness48= verifyNotTree checkSpacefulness "x=0; (( ++x )); echo $x" - -data SpaceStatus = SpaceSome | SpaceNone | SpaceEmpty deriving (Eq) -instance Semigroup SpaceStatus where - SpaceNone <> SpaceNone = SpaceNone - SpaceSome <> _ = SpaceSome - _ <> SpaceSome = SpaceSome - SpaceEmpty <> x = x - x <> SpaceEmpty = x -instance Monoid SpaceStatus where - mempty = SpaceEmpty - mappend = (<>) - --- This is slightly awkward because we want to support structured --- optional checks based on nearly the same logic -checkSpacefulness params = checkSpacefulness' onFind params - where - emit x = tell [x] - onFind spaces token _ = - when (spaces /= SpaceNone) $ - if isDefaultAssignment (parentMap params) token - then - emit $ makeComment InfoC (getId token) 2223 - "This default assignment may cause DoS due to globbing. Quote it." - else - unless (quotesMayConflictWithSC2281 params token) $ - emit $ makeCommentWithFix InfoC (getId token) 2086 - "Double quote to prevent globbing and word splitting." - (addDoubleQuotesAround params token) - - isDefaultAssignment parents token = - let modifier = getBracedModifier $ bracedString token in - any (`isPrefixOf` modifier) ["=", ":="] - && isParamTo parents ":" token - - -- Given a T_DollarBraced, return a simplified version of the string contents. - bracedString (T_DollarBraced _ _ l) = concat $ oversimplify l - bracedString _ = error "Internal shellcheck error, please report! (bracedString on non-variable)" - -prop_checkSpacefulness4v= verifyTree checkVerboseSpacefulness "foo=3; foo=$(echo $foo)" -prop_checkSpacefulness8v= verifyTree checkVerboseSpacefulness "a=foo\\ bar; a=foo; rm $a" -prop_checkSpacefulness28v = verifyTree checkVerboseSpacefulness "exec {n}>&1; echo $n" -prop_checkSpacefulness36v = verifyTree checkVerboseSpacefulness "arg=$#; echo $arg" -prop_checkSpacefulness44v = verifyNotTree checkVerboseSpacefulness "foo=3; $foo=4" -checkVerboseSpacefulness params = checkSpacefulness' onFind params - where - onFind spaces token name = - when (spaces == SpaceNone - && name `notElem` specialVariablesWithoutSpaces - && not (quotesMayConflictWithSC2281 params token)) $ - tell [makeCommentWithFix StyleC (getId token) 2248 - "Prefer double quoting even when variables don't contain special characters." - (addDoubleQuotesAround params token)] - -- Don't suggest quotes if this will instead be autocorrected -- from $foo=bar to foo=bar. This is not pretty but ok. quotesMayConflictWithSC2281 params t = @@ -2121,74 +2018,111 @@ quotesMayConflictWithSC2281 params t = _ -> False addDoubleQuotesAround params token = (surroundWith (getId token) params "\"") -checkSpacefulness' - :: (SpaceStatus -> Token -> String -> Writer [TokenComment] ()) -> - Parameters -> Token -> [TokenComment] -checkSpacefulness' onFind params t = - doVariableFlowAnalysis readF writeF (Map.fromList defaults) (variableFlow params) + +prop_checkSpacefulnessCfg1 = verify checkSpacefulnessCfg "a='cow moo'; echo $a" +prop_checkSpacefulnessCfg2 = verifyNot checkSpacefulnessCfg "a='cow moo'; [[ $a ]]" +prop_checkSpacefulnessCfg3 = verifyNot checkSpacefulnessCfg "a='cow*.mp3'; echo \"$a\"" +prop_checkSpacefulnessCfg4 = verify checkSpacefulnessCfg "for f in *.mp3; do echo $f; done" +prop_checkSpacefulnessCfg4a= verifyNot checkSpacefulnessCfg "foo=3; foo=$(echo $foo)" +prop_checkSpacefulnessCfg5 = verify checkSpacefulnessCfg "a='*'; b=$a; c=lol${b//foo/bar}; echo $c" +prop_checkSpacefulnessCfg6 = verify checkSpacefulnessCfg "a=foo$(lol); echo $a" +prop_checkSpacefulnessCfg7 = verify checkSpacefulnessCfg "a=foo\\ bar; rm $a" +prop_checkSpacefulnessCfg8 = verifyNot checkSpacefulnessCfg "a=foo\\ bar; a=foo; rm $a" +prop_checkSpacefulnessCfg10= verify checkSpacefulnessCfg "rm $1" +prop_checkSpacefulnessCfg11= verify checkSpacefulnessCfg "rm ${10//foo/bar}" +prop_checkSpacefulnessCfg12= verifyNot checkSpacefulnessCfg "(( $1 + 3 ))" +prop_checkSpacefulnessCfg13= verifyNot checkSpacefulnessCfg "if [[ $2 -gt 14 ]]; then true; fi" +prop_checkSpacefulnessCfg14= verifyNot checkSpacefulnessCfg "foo=$3 env" +prop_checkSpacefulnessCfg15= verifyNot checkSpacefulnessCfg "local foo=$1" +prop_checkSpacefulnessCfg16= verifyNot checkSpacefulnessCfg "declare foo=$1" +prop_checkSpacefulnessCfg17= verify checkSpacefulnessCfg "echo foo=$1" +prop_checkSpacefulnessCfg18= verifyNot checkSpacefulnessCfg "$1 --flags" +prop_checkSpacefulnessCfg19= verify checkSpacefulnessCfg "echo $PWD" +prop_checkSpacefulnessCfg20= verifyNot checkSpacefulnessCfg "n+='foo bar'" +prop_checkSpacefulnessCfg21= verifyNot checkSpacefulnessCfg "select foo in $bar; do true; done" +prop_checkSpacefulnessCfg22= verifyNot checkSpacefulnessCfg "echo $\"$1\"" +prop_checkSpacefulnessCfg23= verifyNot checkSpacefulnessCfg "a=(1); echo ${a[@]}" +prop_checkSpacefulnessCfg24= verify checkSpacefulnessCfg "a='a b'; cat <<< $a" +prop_checkSpacefulnessCfg25= verify checkSpacefulnessCfg "a='s/[0-9]//g'; sed $a" +prop_checkSpacefulnessCfg26= verify checkSpacefulnessCfg "a='foo bar'; echo {1,2,$a}" +prop_checkSpacefulnessCfg27= verifyNot checkSpacefulnessCfg "echo ${a:+'foo'}" +prop_checkSpacefulnessCfg28= verifyNot checkSpacefulnessCfg "exec {n}>&1; echo $n" +prop_checkSpacefulnessCfg29= verifyNot checkSpacefulnessCfg "n=$(stuff); exec {n}>&-;" +prop_checkSpacefulnessCfg30= verify checkSpacefulnessCfg "file='foo bar'; echo foo > $file;" +prop_checkSpacefulnessCfg31= verifyNot checkSpacefulnessCfg "echo \"`echo \\\"$1\\\"`\"" +prop_checkSpacefulnessCfg32= verifyNot checkSpacefulnessCfg "var=$1; [ -v var ]" +prop_checkSpacefulnessCfg33= verify checkSpacefulnessCfg "for file; do echo $file; done" +prop_checkSpacefulnessCfg34= verify checkSpacefulnessCfg "declare foo$n=$1" +prop_checkSpacefulnessCfg35= verifyNot checkSpacefulnessCfg "echo ${1+\"$1\"}" +prop_checkSpacefulnessCfg36= verifyNot checkSpacefulnessCfg "arg=$#; echo $arg" +prop_checkSpacefulnessCfg37= verifyNot checkSpacefulnessCfg "@test 'status' {\n [ $status -eq 0 ]\n}" +prop_checkSpacefulnessCfg37v = verify checkVerboseSpacefulnessCfg "@test 'status' {\n [ $status -eq 0 ]\n}" +prop_checkSpacefulnessCfg38= verify checkSpacefulnessCfg "a=; echo $a" +prop_checkSpacefulnessCfg39= verifyNot checkSpacefulnessCfg "a=''\"\"''; b=x$a; echo $b" +prop_checkSpacefulnessCfg40= verifyNot checkSpacefulnessCfg "a=$((x+1)); echo $a" +prop_checkSpacefulnessCfg41= verifyNot checkSpacefulnessCfg "exec $1 --flags" +prop_checkSpacefulnessCfg42= verifyNot checkSpacefulnessCfg "run $1 --flags" +prop_checkSpacefulnessCfg43= verifyNot checkSpacefulnessCfg "$foo=42" +prop_checkSpacefulnessCfg44= verify checkSpacefulnessCfg "#!/bin/sh\nexport var=$value" +prop_checkSpacefulnessCfg45= verifyNot checkSpacefulnessCfg "wait -zzx -p foo; echo $foo" +prop_checkSpacefulnessCfg46= verifyNot checkSpacefulnessCfg "x=0; (( x += 1 )); echo $x" +prop_checkSpacefulnessCfg47= verifyNot checkSpacefulnessCfg "x=0; (( x-- )); echo $x" +prop_checkSpacefulnessCfg48= verifyNot checkSpacefulnessCfg "x=0; (( ++x )); echo $x" +prop_checkSpacefulnessCfg49= verifyNot checkSpacefulnessCfg "for i in 1 2 3; do echo $i; done" +prop_checkSpacefulnessCfg50= verify checkSpacefulnessCfg "for i in 1 2 *; do echo $i; done" +prop_checkSpacefulnessCfg51= verify checkSpacefulnessCfg "x='foo bar'; x && x=1; echo $x" +prop_checkSpacefulnessCfg52= verifyNot checkSpacefulnessCfg "x=1; if f; then x='foo bar'; exit; fi; echo $x" +prop_checkSpacefulnessCfg53= verifyNot checkSpacefulnessCfg "s=1; f() { local s='a b'; }; f; echo $s" +prop_checkSpacefulnessCfg54= verifyNot checkSpacefulnessCfg "s='a b'; f() { s=1; }; f; echo $s" +prop_checkSpacefulnessCfg55= verify checkSpacefulnessCfg "s='a b'; x && f() { s=1; }; f; echo $s" +prop_checkSpacefulnessCfg56= verifyNot checkSpacefulnessCfg "s=1; cat <(s='a b'); echo $s" + +checkSpacefulnessCfg = checkSpacefulnessCfg' True +checkVerboseSpacefulnessCfg = checkSpacefulnessCfg' False + +checkSpacefulnessCfg' :: Bool -> (Parameters -> Token -> Writer [TokenComment] ()) +checkSpacefulnessCfg' dirtyPass params token@(T_DollarBraced id _ list) = + when (needsQuoting && (dirtyPass == not isClean)) $ + unless (name `elem` specialVariablesWithoutSpaces || quotesMayConflictWithSC2281 params token) $ + if dirtyPass + then + if isDefaultAssignment (parentMap params) token + then + info (getId token) 2223 + "This default assignment may cause DoS due to globbing. Quote it." + else + infoWithFix id 2086 "Double quote to prevent globbing and word splitting." $ + addDoubleQuotesAround params token + else + styleWithFix id 2248 "Prefer double quoting even when variables don't contain special characters." $ + addDoubleQuotesAround params token + where - defaults = zip variablesWithoutSpaces (repeat SpaceNone) - - hasSpaces name = gets (Map.findWithDefault SpaceSome name) - - setSpaces name status = - modify $ Map.insert name status - - readF _ token name = do - spaces <- hasSpaces name - let needsQuoting = - isExpansion token - && not (isArrayExpansion token) -- There's another warning for this - && not (isCountingReference token) - && not (isQuoteFree (shellType params) parents token) - && not (isQuotedAlternativeReference token) - && not (usedAsCommandName parents token) - - return . execWriter $ when needsQuoting $ onFind spaces token name - - where - emit x = tell [x] - - writeF _ _ name (DataString SourceExternal) = setSpaces name SpaceSome >> return [] - writeF _ _ name (DataString SourceInteger) = setSpaces name SpaceNone >> return [] - - writeF _ _ name (DataString (SourceFrom vals)) = do - map <- get - setSpaces name - (isSpacefulWord (\x -> Map.findWithDefault SpaceSome x map) vals) - return [] - - writeF _ _ _ _ = return [] - + name = getBracedReference $ concat $ oversimplify list parents = parentMap params + needsQuoting = + not (isArrayExpansion token) -- There's another warning for this + && not (isCountingReference token) + && not (isQuoteFree (shellType params) parents token) + && not (isQuotedAlternativeReference token) + && not (usedAsCommandName parents token) - isExpansion t = - case t of - (T_DollarBraced _ _ _ ) -> True - _ -> False + isClean = fromMaybe False $ do + state <- CF.getIncomingState (cfgAnalysis params) id + value <- Map.lookup name $ CF.variablesInScope state + return $ CF.spaceStatus value == CF.SpaceStatusClean + + isDefaultAssignment parents token = + let modifier = getBracedModifier $ bracedString token in + any (`isPrefixOf` modifier) ["=", ":="] + && isParamTo parents ":" token + + -- Given a T_DollarBraced, return a simplified version of the string contents. + bracedString (T_DollarBraced _ _ l) = concat $ oversimplify l + bracedString _ = error $ pleaseReport "bracedString on non-variable" + +checkSpacefulnessCfg' _ _ _ = return () - isSpacefulWord :: (String -> SpaceStatus) -> [Token] -> SpaceStatus - isSpacefulWord f = mconcat . map (isSpaceful f) - isSpaceful :: (String -> SpaceStatus) -> Token -> SpaceStatus - isSpaceful spacefulF x = - case x of - T_DollarExpansion _ _ -> SpaceSome - T_Backticked _ _ -> SpaceSome - T_Glob _ _ -> SpaceSome - T_Extglob {} -> SpaceSome - T_DollarArithmetic _ _ -> SpaceNone - T_Literal _ s -> fromLiteral s - T_SingleQuoted _ s -> fromLiteral s - T_DollarBraced _ _ l -> spacefulF $ getBracedReference $ concat $ oversimplify l - T_NormalWord _ w -> isSpacefulWord spacefulF w - T_DoubleQuoted _ w -> isSpacefulWord spacefulF w - _ -> SpaceEmpty - where - globspace = "*?[] \t\n" - containsAny s = any (`elem` s) - fromLiteral "" = SpaceEmpty - fromLiteral s | s `containsAny` globspace = SpaceSome - fromLiteral _ = SpaceNone prop_CheckVariableBraces1 = verify checkVariableBraces "a='123'; echo $a" prop_CheckVariableBraces2 = verifyNot checkVariableBraces "a='123'; echo ${a}" From 8dc0fdb4cc3ae7cdea693b5dd2bead54fad7422e Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 20 Jul 2022 12:43:28 -0700 Subject: [PATCH 241/459] Precompile new fgl dependency on armv6hf --- build/linux.armv6hf/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/build/linux.armv6hf/Dockerfile b/build/linux.armv6hf/Dockerfile index bd5795c..f933dda 100644 --- a/build/linux.armv6hf/Dockerfile +++ b/build/linux.armv6hf/Dockerfile @@ -52,6 +52,7 @@ RUN pirun apt-get install -y ghc cabal-install ENV CABALOPTS "--ghc-options;-split-sections -optc-Os -optc-Wl,--gc-sections;--gcc-options;-Os -Wl,--gc-sections -ffunction-sections -fdata-sections" RUN pirun cabal update RUN IFS=";" && pirun cabal install --dependencies-only $CABALOPTS ShellCheck +RUN IFS=';' && pirun cabal install $CABALOPTS --lib fgl # Copy the build script WORKDIR /pi/scratch From 3ee4419ef4f4cda211401b1e97898dda5eb2e684 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 21 Jul 2022 15:06:05 -0700 Subject: [PATCH 242/459] Suppress SC2086 for variables declared -i (ref #2541) --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 16 +- src/ShellCheck/CFG.hs | 102 ++++++---- src/ShellCheck/CFGAnalysis.hs | 365 ++++++++++++++++++++++++++-------- 4 files changed, 358 insertions(+), 126 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 66d5369..7da6077 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Fixed - SC2086: Now uses DFA to make more accurate predictions about values +- SC2086: No longer warns about values declared as integer with declare -i ### Changed - ShellCheck now has a Data Flow Analysis engine to make smarter decisions diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 0062879..ca918ba 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -24,6 +24,7 @@ module ShellCheck.Analytics (runAnalytics, optionalChecks, ShellCheck.Analytics. import ShellCheck.AST import ShellCheck.ASTLib import ShellCheck.AnalyzerLib hiding (producesComments) +import ShellCheck.CFG import qualified ShellCheck.CFGAnalysis as CF import ShellCheck.Data import ShellCheck.Parser @@ -46,6 +47,7 @@ import Data.Ord import Data.Semigroup import Debug.Trace -- STRIP import qualified Data.Map.Strict as Map +import qualified Data.Set as S import Test.QuickCheck.All (forAllProperties) import Test.QuickCheck.Test (quickCheckWithResult, stdArgs, maxSuccess) @@ -2076,6 +2078,14 @@ prop_checkSpacefulnessCfg53= verifyNot checkSpacefulnessCfg "s=1; f() { local s= prop_checkSpacefulnessCfg54= verifyNot checkSpacefulnessCfg "s='a b'; f() { s=1; }; f; echo $s" prop_checkSpacefulnessCfg55= verify checkSpacefulnessCfg "s='a b'; x && f() { s=1; }; f; echo $s" prop_checkSpacefulnessCfg56= verifyNot checkSpacefulnessCfg "s=1; cat <(s='a b'); echo $s" +prop_checkSpacefulnessCfg57= verifyNot checkSpacefulnessCfg "declare -i s=0; s=$(f); echo $s" +prop_checkSpacefulnessCfg58= verify checkSpacefulnessCfg "f() { declare -i s; }; f; s=$(var); echo $s" +prop_checkSpacefulnessCfg59= verifyNot checkSpacefulnessCfg "f() { declare -gi s; }; f; s=$(var); echo $s" +prop_checkSpacefulnessCfg60= verify checkSpacefulnessCfg "declare -i s; declare +i s; s=$(foo); echo $s" +prop_checkSpacefulnessCfg61= verify checkSpacefulnessCfg "declare -x X; y=foo$X; echo $y;" +prop_checkSpacefulnessCfg62= verifyNot checkSpacefulnessCfg "f() { declare -x X; y=foo$X; echo $y; }" +prop_checkSpacefulnessCfg63= verify checkSpacefulnessCfg "f && declare -i s; s='x + y'; echo $s" +prop_checkSpacefulnessCfg64= verifyNot checkSpacefulnessCfg "declare -i s; s='x + y'; x=$s; echo $x" checkSpacefulnessCfg = checkSpacefulnessCfg' True checkVerboseSpacefulnessCfg = checkSpacefulnessCfg' False @@ -2110,7 +2120,11 @@ checkSpacefulnessCfg' dirtyPass params token@(T_DollarBraced id _ list) = isClean = fromMaybe False $ do state <- CF.getIncomingState (cfgAnalysis params) id value <- Map.lookup name $ CF.variablesInScope state - return $ CF.spaceStatus value == CF.SpaceStatusClean + return $ isCleanState value + + isCleanState state = + (all (S.member CFVPInteger) $ CF.variableProperties state) + || CF.spaceStatus (CF.variableValue state) == CF.SpaceStatusClean isDefaultAssignment parents token = let modifier = getBracedModifier $ bracedString token in diff --git a/src/ShellCheck/CFG.hs b/src/ShellCheck/CFG.hs index 101a0d7..a4bd166 100644 --- a/src/ShellCheck/CFG.hs +++ b/src/ShellCheck/CFG.hs @@ -32,6 +32,7 @@ module ShellCheck.CFG ( CFGraph, CFGParameters (..), IdTagged (..), + Scope (..), buildGraph , ShellCheck.CFG.runTests -- STRIP ) @@ -105,7 +106,8 @@ data CFEdge = -- Actions we track data CFEffect = - CFModifyProps String [CFVariableProp] + CFSetProps Scope String (S.Set CFVariableProp) + | CFUnsetProps Scope String (S.Set CFVariableProp) | CFReadVariable String | CFWriteVariable String CFValue | CFWriteGlobal String CFValue @@ -143,7 +145,7 @@ data CFValue = data CFStringPart = -- A known literal string value, like 'foo' CFStringLiteral String - -- The contents of a variable, like $foo + -- The contents of a variable, like $foo (may not be a string) | CFStringVariable String -- An value that is unknown but an integer | CFStringInteger @@ -152,7 +154,7 @@ data CFStringPart = deriving (Eq, Ord, Show, Generic, NFData) -- The properties of a variable -data CFVariableProp = CFVPExport | CFVPArray +data CFVariableProp = CFVPExport | CFVPArray | CFVPAssociative | CFVPInteger deriving (Eq, Ord, Show, Generic, NFData) -- Options when generating CFG @@ -961,71 +963,92 @@ handleCommand cmd vars args literalCmd = do handleDeclare (cmd:args) = do isFunc <- asks cfIsFunction - let (evaluated, effects) = mconcat $ map (toEffects isFunc) args + -- This is a bit of a kludge: we don't have great support for things like + -- 'declare -i x=$x' so do one round with declare x=$x, followed by declare -i x + let (evaluated, assignments, added, removed) = mconcat $ map (toEffects isFunc) args before <- sequentially $ evaluated - effect <- newNodeRange $ CFApplyEffects effects + assignments <- newNodeRange $ CFApplyEffects assignments + addedProps <- if null added then newStructuralNode else newNodeRange $ CFApplyEffects added + removedProps <- if null removed then newStructuralNode else newNodeRange $ CFApplyEffects removed result <- newNodeRange $ CFSetExitCode (getId cmd) - linkRanges [before, effect, result] + linkRanges [before, assignments, addedProps, removedProps, result] where opts = map fst $ getGenericOpts args - array = "a" `elem` opts || "A" `elem` opts + array = "a" `elem` opts || associative + associative = "A" `elem` opts integer = "i" `elem` opts func = "f" `elem` opts || "F" `elem` opts global = "g" `elem` opts + export = "x" `elem` opts writer isFunc = case () of _ | global -> CFWriteGlobal _ | isFunc -> CFWriteLocal _ -> CFWriteVariable - toEffects :: Bool -> Token -> ([Token], [IdTagged CFEffect]) + scope isFunc = + case () of + _ | global -> GlobalScope + _ | isFunc -> LocalScope + _ -> DefaultScope + + addedProps = S.fromList $ concat $ [ + [ CFVPArray | array ], + [ CFVPInteger | integer ], + [ CFVPExport | export ], + [ CFVPAssociative | associative ] + ] + + removedProps = S.fromList $ concat $ [ + -- Array property can't be unset + [ CFVPInteger | 'i' `elem` unsetOptions ], + [ CFVPExport | 'e' `elem` unsetOptions ] + ] + toEffects isFunc (T_Assignment id mode var idx t) = let pre = idx ++ [t] - isArray = array || (not $ null idx) - asArray = [ IdTagged id $ (writer isFunc) var CFValueArray ] - asString = [ IdTagged id $ (writer isFunc) var $ - if integer - then CFValueInteger -- TODO: Also handle integer variable property - else CFValueComputed (getId t) $ [ CFStringVariable var | mode == Append ] ++ tokenToParts t - ] + val = [ IdTagged id $ (writer isFunc) var $ CFValueComputed (getId t) $ [ CFStringVariable var | mode == Append ] ++ tokenToParts t ] + added = [ IdTagged id $ CFSetProps (scope isFunc) var addedProps | not $ S.null addedProps ] + removed = [ IdTagged id $ CFUnsetProps (scope isFunc) var addedProps | not $ S.null removedProps ] in - (pre, if isArray then asArray else asString ) + (pre, val, added, removed) toEffects isFunc t = let + id = getId t pre = [t] literal = fromJust $ getLiteralStringExt (const $ Just "\0") t isKnown = '\0' `notElem` literal match = fmap head $ variableAssignRegex `matchRegex` literal name = fromMaybe literal match - typer def = - if array - then CFValueArray - else - if integer - then CFValueInteger - else def + asLiteral = + IdTagged id $ (writer isFunc) name $ + CFValueComputed (getId t) [ CFStringLiteral $ drop 1 $ dropWhile (/= '=') $ literal ] + asUnknown = + IdTagged id $ (writer isFunc) name $ + CFValueString + + added = [ IdTagged id $ CFSetProps (scope isFunc) name addedProps ] + removed = [ IdTagged id $ CFUnsetProps (scope isFunc) name removedProps ] - asLiteral = [ - IdTagged (getId t) $ (writer isFunc) name $ - typer $ CFValueComputed (getId t) [ CFStringLiteral $ drop 1 $ dropWhile (/= '=') $ literal ] - ] - asUnknown = [ - IdTagged (getId t) $ (writer isFunc) name $ - typer $ CFValueString - ] - asBlank = [ - IdTagged (getId t) $ (writer isFunc) name $ - typer $ CFValueComputed (getId t) [] - ] in case () of - _ | not (isVariableName name) -> (pre, []) - _ | isJust match && isKnown -> (pre, asLiteral) - _ | isJust match -> (pre, asUnknown) - _ -> (pre, asBlank) + _ | not (isVariableName name) -> (pre, [], [], []) + _ | isJust match && isKnown -> (pre, [asLiteral], added, removed) + _ | isJust match -> (pre, [asUnknown], added, removed) + -- e.g. declare -i x + _ -> (pre, [], added, removed) + + -- find "ia" from `define +i +a` + unsetOptions :: String + unsetOptions = + let + strings = mapMaybe getLiteralString args + plusses = filter ("+" `isPrefixOf`) strings + in + concatMap (drop 1) plusses handlePrintf (cmd:args) = newNodeRange $ CFApplyEffects $ maybeToList findVar @@ -1103,6 +1126,7 @@ handleCommand cmd vars args literalCmd = do none = newStructuralNode data Scope = DefaultScope | GlobalScope | LocalScope | PrefixScope + deriving (Eq, Ord, Show, Generic, NFData) buildAssignment scope t = do op <- case t of diff --git a/src/ShellCheck/CFGAnalysis.hs b/src/ShellCheck/CFGAnalysis.hs index 99ce450..0007a67 100644 --- a/src/ShellCheck/CFGAnalysis.hs +++ b/src/ShellCheck/CFGAnalysis.hs @@ -50,7 +50,9 @@ module ShellCheck.CFGAnalysis ( ,CFGParameters (..) ,CFGAnalysis (..) ,ProgramState (..) + ,VariableState (..) ,VariableValue (..) + ,VariableProperties ,SpaceStatus (..) ,getIncomingState ,getOutgoingState @@ -77,9 +79,21 @@ import Debug.Trace -- STRIP import Test.QuickCheck +-- The number of iterations for DFA to stabilize iterationCount = 1000000 +-- There have been multiple bugs where bad caching caused oscillations. +-- As a precaution, disable caching if there's this many iterations left. +fallbackThreshold = 10000 +-- The number of cache entries to keep per node cacheEntries = 10 +logVerbose log = do + -- traceShowM log + return () +logInfo log = do + -- traceShowM log + return () + -- The result of the data flow analysis data CFGAnalysis = CFGAnalysis { graph :: CFGraph, @@ -89,9 +103,9 @@ data CFGAnalysis = CFGAnalysis { -- The program state we expose externally data ProgramState = ProgramState { - variablesInScope :: M.Map String VariableValue, +-- internalState :: InternalState, -- For debugging + variablesInScope :: M.Map String VariableState, stateIsReachable :: Bool --- internalState :: InternalState } deriving (Show, Eq, Generic, NFData) -- Conveniently get the state before a token id @@ -111,9 +125,9 @@ getDataForNode analysis node = M.lookup node $ nodeToData analysis -- The current state of data flow at a point in the program, potentially as a diff data InternalState = InternalState { sVersion :: Integer, - sGlobalValues :: VersionedMap String VariableValue, - sLocalValues :: VersionedMap String VariableValue, - sPrefixValues :: VersionedMap String VariableValue, + sGlobalValues :: VersionedMap String VariableState, + sLocalValues :: VersionedMap String VariableState, + sPrefixValues :: VersionedMap String VariableState, sFunctionTargets :: VersionedMap String FunctionValue, sIsReachable :: Maybe Bool } deriving (Show, Generic, NFData) @@ -135,31 +149,33 @@ unreachableState = modified newInternalState { createEnvironmentState :: InternalState createEnvironmentState = do foldl' (flip ($)) newInternalState $ concat [ - addVars Data.internalVariables unknownVariableValue, - addVars Data.variablesWithoutSpaces spacelessVariableValue, - addVars Data.specialIntegerVariables spacelessVariableValue + addVars Data.internalVariables unknownVariableState, + addVars Data.variablesWithoutSpaces spacelessVariableState, + addVars Data.specialIntegerVariables spacelessVariableState ] where addVars names val = map (\name -> insertGlobal name val) names - spacelessVariableValue = VariableValue { - literalValue = Nothing, - spaceStatus = SpaceStatusClean + spacelessVariableState = unknownVariableState { + variableValue = VariableValue { + literalValue = Nothing, + spaceStatus = SpaceStatusClean + } } modified s = s { sVersion = -1 } -insertGlobal :: String -> VariableValue -> InternalState -> InternalState +insertGlobal :: String -> VariableState -> InternalState -> InternalState insertGlobal name value state = modified state { sGlobalValues = vmInsert name value $ sGlobalValues state } -insertLocal :: String -> VariableValue -> InternalState -> InternalState +insertLocal :: String -> VariableState -> InternalState -> InternalState insertLocal name value state = modified state { sLocalValues = vmInsert name value $ sLocalValues state } -insertPrefix :: String -> VariableValue -> InternalState -> InternalState +insertPrefix :: String -> VariableState -> InternalState -> InternalState insertPrefix name value state = modified state { sPrefixValues = vmInsert name value $ sPrefixValues state } @@ -169,24 +185,38 @@ insertFunction name value state = modified state { sFunctionTargets = vmInsert name value $ sFunctionTargets state } +addProperties :: S.Set CFVariableProp -> VariableState -> VariableState +addProperties props state = state { + variableProperties = S.map (S.union props) $ variableProperties state +} + +removeProperties :: S.Set CFVariableProp -> VariableState -> VariableState +removeProperties props state = state { + variableProperties = S.map (\s -> S.difference s props) $ variableProperties state +} + internalToExternal :: InternalState -> ProgramState internalToExternal s = ProgramState { - -- Avoid introducing dependencies on the literal value as this is only for debugging purposes right now - variablesInScope = M.map (\c -> c { literalValue = Nothing }) flatVars, + -- Censor the literal value to avoid introducing dependencies on it. It's just for debugging. + variablesInScope = M.map censor flatVars, -- internalState = s, -- For debugging stateIsReachable = fromMaybe True $ sIsReachable s } where + censor s = s { + variableValue = (variableValue s) { + literalValue = Nothing + } + } flatVars = M.unionsWith (\_ last -> last) $ map mapStorage [sGlobalValues s, sLocalValues s, sPrefixValues s] -- Dependencies on values, e.g. "if there is a global variable named 'foo' without spaces" -- This is used to see if the DFA of a function would result in the same state, so anything -- that affects DFA must be tracked. data StateDependency = - DepGlobalValue String VariableValue - | DepLocalValue String VariableValue - | DepPrefixValue String VariableValue + DepState Scope String VariableState + | DepProperties Scope String VariableProperties | DepFunction String (S.Set FunctionDefinition) -- Whether invoking the node would result in recursion (i.e., is the function on the stack?) | DepIsRecursive Node Bool @@ -199,10 +229,6 @@ data FunctionDefinition = FunctionUnknown | FunctionDefinition String Node Node -- The Set of places a command name can point (it's a Set to handle conditionally defined functions) type FunctionValue = S.Set FunctionDefinition --- The scope of a function. ("Prefix" refers to e.g. `foo=1 env`) -data VariableScope = PrefixVar | LocalVar | GlobalVar - deriving (Show, Eq, Ord, Generic, NFData) - -- Create an InternalState that fulfills the given dependencies depsToState :: S.Set StateDependency -> InternalState depsToState set = foldl insert newInternalState $ S.toList set @@ -211,11 +237,26 @@ depsToState set = foldl insert newInternalState $ S.toList set insert state dep = case dep of DepFunction name val -> insertFunction name val state - DepGlobalValue name val -> insertGlobal name val state - DepLocalValue name val -> insertLocal name val state - DepPrefixValue name val -> insertPrefix name val state + DepState scope name val -> insertIn True scope name val state + -- State includes properties and more, so don't overwrite a state with properties + DepProperties scope name props -> insertIn False scope name unknownVariableState { variableProperties = props } state DepIsRecursive _ _ -> state + insertIn overwrite scope name val state = + let + (mapToCheck, inserter) = + case scope of + PrefixScope -> (sPrefixValues, insertPrefix) + LocalScope -> (sLocalValues, insertLocal) + GlobalScope -> (sGlobalValues, insertGlobal) + DefaultScope -> error $ pleaseReport "Unresolved scope in dependency" + + alreadyExists = isJust $ vmLookup name $ mapToCheck state + in + if overwrite || not alreadyExists + then inserter name val state + else state + unknownFunctionValue = S.singleton FunctionUnknown -- The information about the value of a single variable @@ -225,20 +266,45 @@ data VariableValue = VariableValue { } deriving (Show, Eq, Ord, Generic, NFData) +data VariableState = VariableState { + variableValue :: VariableValue, + variableProperties :: VariableProperties +} + deriving (Show, Eq, Ord, Generic, NFData) + -- Whether or not the value needs quoting (has spaces/globs), or we don't know data SpaceStatus = SpaceStatusEmpty | SpaceStatusClean | SpaceStatusDirty deriving (Show, Eq, Ord, Generic, NFData) +-- The set of possible sets of properties for this variable +type VariableProperties = S.Set (S.Set CFVariableProp) + +defaultProperties = S.singleton S.empty + +unknownVariableState = VariableState { + variableValue = unknownVariableValue, + variableProperties = defaultProperties +} unknownVariableValue = VariableValue { literalValue = Nothing, spaceStatus = SpaceStatusDirty } -emptyVariableValue = VariableValue { +emptyVariableValue = unknownVariableValue { literalValue = Just "", spaceStatus = SpaceStatusEmpty } +unsetVariableState = VariableState { + variableValue = emptyVariableValue, + variableProperties = defaultProperties +} + +mergeVariableState a b = VariableState { + variableValue = mergeVariableValue (variableValue a) (variableValue b), + variableProperties = S.union (variableProperties a) (variableProperties b) +} + mergeVariableValue a b = VariableValue { literalValue = if literalValue a == literalValue b then literalValue a else Nothing, spaceStatus = mergeSpaceStatus (spaceStatus a) (spaceStatus b) @@ -296,6 +362,8 @@ data Ctx s = Ctx { cCounter :: STRef s Integer, -- A cache of input state dependencies to output effects cCache :: STRef s (M.Map Node [(S.Set StateDependency, InternalState)]), + -- Whether the cache is enabled (see fallbackThreshold) + cEnableCache :: STRef s Bool, -- The states resulting from data flows per invocation path cInvocations :: STRef s (M.Map [Node] (S.Set StateDependency, M.Map Node (InternalState, InternalState))) } @@ -304,6 +372,8 @@ data Ctx s = Ctx { data StackEntry s = StackEntry { -- The entry point of this stack entry for the purpose of detecting recursion entryPoint :: Node, + -- Whether this is a function call (as opposed to a subshell) + isFunctionCall :: Bool, -- The node where this entry point was invoked callSite :: Node, -- A mutable set of dependencies we fetched from here or higher in the stack @@ -369,9 +439,9 @@ mergeState ctx a b = do return unreachableState _ | sVersion a >= 0 && sVersion b >= 0 && sVersion a == sVersion b -> return a _ -> do - globals <- mergeMaps ctx mergeVariableValue readGlobal (sGlobalValues a) (sGlobalValues b) - locals <- mergeMaps ctx mergeVariableValue readVariable (sLocalValues a) (sLocalValues b) - prefix <- mergeMaps ctx mergeVariableValue readVariable (sPrefixValues a) (sPrefixValues b) + globals <- mergeMaps ctx mergeVariableState readGlobal (sGlobalValues a) (sGlobalValues b) + locals <- mergeMaps ctx mergeVariableState readVariable (sLocalValues a) (sLocalValues b) + prefix <- mergeMaps ctx mergeVariableState readVariable (sPrefixValues a) (sPrefixValues b) funcs <- mergeMaps ctx S.union readFunction (sFunctionTargets a) (sFunctionTargets b) return $ InternalState { sVersion = -1, @@ -517,15 +587,15 @@ vmPatch base diff = mapStorage = M.unionWith (flip const) (mapStorage base) (mapStorage diff) } --- Modify a variable as with x=1. This applies it to the appropriate scope. -writeVariable :: forall s. Ctx s -> String -> VariableValue -> ST s () +-- Set a variable. This includes properties. Applies it to the appropriate scope. +writeVariable :: forall s. Ctx s -> String -> VariableState -> ST s () writeVariable ctx name val = do - (_, typ) <- readVariableWithScope ctx name + typ <- readVariableScope ctx name case typ of - GlobalVar -> writeGlobal ctx name val - LocalVar -> writeLocal ctx name val + GlobalScope -> writeGlobal ctx name val + LocalScope -> writeLocal ctx name val -- Prefixed variables actually become local variables in the invoked function - PrefixVar -> writeLocal ctx name val + PrefixScope -> writeLocal ctx name val writeGlobal ctx name val = do modifySTRef (cOutput ctx) $ insertGlobal name val @@ -536,39 +606,97 @@ writeLocal ctx name val = do writePrefix ctx name val = do modifySTRef (cOutput ctx) $ insertPrefix name val +updateVariableValue ctx name val = do + (props, scope) <- readVariablePropertiesWithScope ctx name + let f = case scope of + GlobalScope -> writeGlobal + LocalScope -> writeLocal + PrefixScope -> writeLocal -- Updates become local + f ctx name $ VariableState { variableValue = val, variableProperties = props } + +updateGlobalValue ctx name val = do + props <- readGlobalProperties ctx name + writeGlobal ctx name VariableState { variableValue = val, variableProperties = props } + +updateLocalValue ctx name val = do + props <- readLocalProperties ctx name + writeLocal ctx name VariableState { variableValue = val, variableProperties = props } + +updatePrefixValue ctx name val = do + -- Prefix variables don't inherit properties + writePrefix ctx name VariableState { variableValue = val, variableProperties = defaultProperties } + + -- Look up a variable value, and also return its scope -readVariableWithScope :: forall s. Ctx s -> String -> ST s (VariableValue, VariableScope) +readVariableWithScope :: forall s. Ctx s -> String -> ST s (VariableState, Scope) readVariableWithScope ctx name = lookupStack get dep def ctx name where - def = (unknownVariableValue, GlobalVar) + def = (unknownVariableState, GlobalScope) get = getVariableWithScope - dep k v = - case v of - (val, GlobalVar) -> DepGlobalValue k val - (val, LocalVar) -> DepLocalValue k val - (val, PrefixVar) -> DepPrefixValue k val + dep k (val, scope) = DepState scope k val -getVariableWithScope :: InternalState -> String -> Maybe (VariableValue, VariableScope) +-- Look up the variable's properties. This can be done independently to avoid incurring a dependency on the value. +readVariablePropertiesWithScope :: forall s. Ctx s -> String -> ST s (VariableProperties, Scope) +readVariablePropertiesWithScope ctx name = lookupStack get dep def ctx name + where + def = (defaultProperties, GlobalScope) + get s k = do + (val, scope) <- getVariableWithScope s k + return (variableProperties val, scope) + dep k (val, scope) = DepProperties scope k val + +readVariableScope ctx name = snd <$> readVariablePropertiesWithScope ctx name + +getVariableWithScope :: InternalState -> String -> Maybe (VariableState, Scope) getVariableWithScope s name = case (vmLookup name $ sPrefixValues s, vmLookup name $ sLocalValues s, vmLookup name $ sGlobalValues s) of - (Just var, _, _) -> return (var, PrefixVar) - (_, Just var, _) -> return (var, LocalVar) - (_, _, Just var) -> return (var, GlobalVar) + (Just var, _, _) -> return (var, PrefixScope) + (_, Just var, _) -> return (var, LocalScope) + (_, _, Just var) -> return (var, GlobalScope) _ -> Nothing undefineFunction ctx name = writeFunction ctx name $ FunctionUnknown undefineVariable ctx name = - writeVariable ctx name $ emptyVariableValue + writeVariable ctx name $ unsetVariableState readVariable ctx name = fst <$> readVariableWithScope ctx name +readVariableProperties ctx name = fst <$> readVariablePropertiesWithScope ctx name readGlobal ctx name = lookupStack get dep def ctx name where - def = unknownVariableValue + def = unknownVariableState -- could come from the environment get s name = vmLookup name $ sGlobalValues s - dep k v = DepGlobalValue k v + dep k v = DepState GlobalScope k v + + +readGlobalProperties ctx name = lookupStack get dep def ctx name + where + def = defaultProperties + get s name = variableProperties <$> (vmLookup name $ sGlobalValues s) + -- This dependency will fail to match if it's shadowed by a local variable, + -- such as in x=1; f() { local -i x; declare -ag x; } because we'll look at + -- x and find it to be local and not global. FIXME? + dep k v = DepProperties GlobalScope k v + +readLocal ctx name = lookupStackUntilFunction get dep def ctx name + where + def = unsetVariableState -- can't come from the environment + get s name = vmLookup name $ sLocalValues s + dep k v = DepState LocalScope k v + +-- We only want to look up the local properties of the current function, +-- though preferably even if we're in a subshell. FIXME? +readLocalProperties ctx name = fst <$> lookupStackUntilFunction get dep def ctx name + where + def = (defaultProperties, LocalScope) + with tag f = do + val <- variableProperties <$> f + return (val, tag) + + get s name = (with LocalScope $ vmLookup name $ sLocalValues s) `mplus` (with PrefixScope $ vmLookup name $ sPrefixValues s) + dep k (val, scope) = DepProperties scope k val readFunction ctx name = lookupStack get dep def ctx name where @@ -581,9 +709,11 @@ writeFunction ctx name val = do -- Look up each state on the stack until a value is found (or the default is used), -- then add this value as a StateDependency. -lookupStack :: forall s k v. +lookupStack' :: forall s k v. + -- Whether to stop at function boundaries + Bool -- A function that maybe finds a value from a state - (InternalState -> k -> Maybe v) + -> (InternalState -> k -> Maybe v) -- A function that creates a dependency on what was found -> (k -> v -> StateDependency) -- A default value, if the value can't be found anywhere @@ -594,13 +724,14 @@ lookupStack :: forall s k v. -> k -- Returning the result -> ST s v -lookupStack get dep def ctx key = do +lookupStack' functionOnly get dep def ctx key = do top <- readSTRef $ cInput ctx case get top key of Just v -> return v Nothing -> f (cStack ctx) where f [] = return def + f (s:_) | functionOnly && isFunctionCall s = return def f (s:rest) = do -- Go up the stack until we find the value, and add -- a dependency on each state (including where it was found) @@ -608,6 +739,9 @@ lookupStack get dep def ctx key = do modifySTRef (dependencies s) $ S.insert $ dep key res return res +lookupStack = lookupStack' False +lookupStackUntilFunction = lookupStack' True + -- Like lookupStack but without adding dependencies peekStack get def ctx key = do top <- readSTRef $ cInput ctx @@ -621,26 +755,30 @@ peekStack get def ctx key = do Just v -> return v Nothing -> f rest --- Check if the current context fulfills a StateDependency -fulfillsDependency ctx dep = +-- Check if the current context fulfills a StateDependency if entering `entry` +fulfillsDependency ctx entry dep = case dep of - DepGlobalValue name val -> (== (val, GlobalVar)) <$> peek ctx name - DepLocalValue name val -> (== (val, LocalVar)) <$> peek ctx name - DepPrefixValue name val -> (== (val, PrefixVar)) <$> peek ctx name + DepState scope name val -> (== (val, scope)) <$> peek scope ctx name + DepProperties scope name props -> do + (state, s) <- peek scope ctx name + return $ scope == s && variableProperties state == props DepFunction name val -> (== val) <$> peekFunc ctx name + -- Hack. Since we haven't pushed the soon-to-be invoked function on the stack, + -- it won't be found by the normal check. + DepIsRecursive node val | node == entry -> return True DepIsRecursive node val -> return $ val == any (\f -> entryPoint f == node) (cStack ctx) -- _ -> error $ "Unknown dep " ++ show dep where - peek = peekStack getVariableWithScope (unknownVariableValue, GlobalVar) + peek scope = peekStack getVariableWithScope $ if scope == GlobalScope then (unknownVariableState, GlobalScope) else (unsetVariableState, LocalScope) peekFunc = peekStack (\state name -> vmLookup name $ sFunctionTargets state) unknownFunctionValue -- Check if the current context fulfills all StateDependencies -fulfillsDependencies ctx deps = +fulfillsDependencies ctx entry deps = f $ S.toList deps where f [] = return True f (dep:rest) = do - res <- fulfillsDependency ctx dep + res <- fulfillsDependency ctx entry dep if res then f rest else return False @@ -652,6 +790,7 @@ newCtx g = do output <- newSTRef undefined node <- newSTRef undefined cache <- newSTRef M.empty + enableCache <- newSTRef True invocations <- newSTRef M.empty return $ Ctx { cCounter = c, @@ -659,6 +798,7 @@ newCtx g = do cOutput = output, cNode = node, cCache = cache, + cEnableCache = enableCache, cStack = [], cInvocations = invocations, cGraph = g @@ -672,20 +812,21 @@ nextVersion ctx = do return n -- Create a new StackEntry -newStackEntry ctx point = do +newStackEntry ctx point isCall = do deps <- newSTRef S.empty state <- readSTRef $ cOutput ctx callsite <- readSTRef $ cNode ctx return $ StackEntry { entryPoint = point, + isFunctionCall = isCall, callSite = callsite, dependencies = deps, stackState = state } -- Call a function with a new stack entry on the stack -withNewStackFrame ctx node f = do - newEntry <- newStackEntry ctx node +withNewStackFrame ctx node isCall f = do + newEntry <- newStackEntry ctx node isCall newInput <- newSTRef newInternalState newOutput <- newSTRef newInternalState newNode <- newSTRef node @@ -753,7 +894,7 @@ transferSubshell ctx reason entry exit = do writeSTRef cout initial where f entry exit ctx = do - (states, frame) <- withNewStackFrame ctx entry (flip dataflow $ entry) + (states, frame) <- withNewStackFrame ctx entry False (flip dataflow $ entry) let (_, res) = fromMaybe (error $ pleaseReport "Subshell has no exit") $ M.lookup exit states deps <- readSTRef $ dependencies frame registerFlowResult ctx entry states deps @@ -763,12 +904,12 @@ transferSubshell ctx reason entry exit = do transferCommand ctx Nothing = return () transferCommand ctx (Just name) = do targets <- readFunction ctx name - --traceShowM ("Transferring ",name,targets) + logVerbose ("Transferring ",name,targets) transferMultiple ctx $ map (flip transferFunctionValue) $ S.toList targets -- Transfer a set of function definitions and merge the output states. transferMultiple ctx funcs = do --- traceShowM ("Transferring set of ", length funcs) + logVerbose ("Transferring set of ", length funcs) original <- readSTRef out branches <- mapM (apply ctx original) funcs merged <- mergeStates ctx original branches @@ -792,7 +933,7 @@ transferFunctionValue ctx funcVal = else runCached ctx entry (f name entry exit) where f name entry exit ctx = do - (states, frame) <- withNewStackFrame ctx entry (flip dataflow $ entry) + (states, frame) <- withNewStackFrame ctx entry True (flip dataflow $ entry) deps <- readSTRef $ dependencies frame let res = case M.lookup exit states of @@ -827,25 +968,31 @@ runCached ctx node f = do cache <- getCache ctx node case cache of Just v -> do - -- traceShowM $ ("Running cached", node) + logInfo ("Running cached", node) + -- do { (deps, diff) <- f ctx; unless (v == diff) $ traceShowM ("Cache FAILED to match actual result", node, deps, diff); } patchOutputM ctx v + Nothing -> do - -- traceShowM $ ("Cache failed", node) + logInfo ("Cache failed", node) (deps, diff) <- f ctx modifySTRef (cCache ctx) (M.insertWith (\_ old -> (deps, diff):(take cacheEntries old)) node [(deps,diff)]) - -- traceShowM $ ("Recomputed cache for", node, deps) + logVerbose ("Recomputed cache for", node, deps) + -- do { f <- fulfillsDependencies ctx node deps; unless (f) $ traceShowM ("New dependencies FAILED to match", node, deps); } patchOutputM ctx diff -- Get a cached version whose dependencies are currently fulfilled, if any. getCache :: forall s. Ctx s -> Node -> ST s (Maybe InternalState) getCache ctx node = do cache <- readSTRef $ cCache ctx - -- traceShowM $ ("Cache for", node, "length", length $ M.findWithDefault [] node cache, M.lookup node cache) - f $ M.findWithDefault [] node cache + enable <- readSTRef $ cEnableCache ctx + logVerbose ("Cache for", node, "length", length $ M.findWithDefault [] node cache, M.lookup node cache) + if enable + then f $ M.findWithDefault [] node cache + else return Nothing where f [] = return Nothing f ((deps, value):rest) = do - match <- fulfillsDependencies ctx deps + match <- fulfillsDependencies ctx node deps if match then return $ Just value else f rest @@ -857,16 +1004,52 @@ transferEffect ctx effect = void $ readVariable ctx name CFWriteVariable name value -> do val <- cfValueToVariableValue ctx value - writeVariable ctx name val + updateVariableValue ctx name val CFWriteGlobal name value -> do val <- cfValueToVariableValue ctx value - writeGlobal ctx name val + updateGlobalValue ctx name val CFWriteLocal name value -> do val <- cfValueToVariableValue ctx value - writeLocal ctx name val + updateLocalValue ctx name val CFWritePrefix name value -> do val <- cfValueToVariableValue ctx value - writePrefix ctx name val + updatePrefixValue ctx name val + + CFSetProps scope name props -> + case scope of + DefaultScope -> do + state <- readVariable ctx name + writeVariable ctx name $ addProperties props state + GlobalScope -> do + state <- readGlobal ctx name + writeGlobal ctx name $ addProperties props state + LocalScope -> do + out <- readSTRef (cOutput ctx) + state <- readLocal ctx name + writeLocal ctx name $ addProperties props state + PrefixScope -> do + -- Prefix values become local + state <- readLocal ctx name + writeLocal ctx name $ addProperties props state + + CFUnsetProps scope name props -> + case scope of + DefaultScope -> do + state <- readVariable ctx name + writeVariable ctx name $ removeProperties props state + GlobalScope -> do + state <- readGlobal ctx name + writeGlobal ctx name $ removeProperties props state + LocalScope -> do + out <- readSTRef (cOutput ctx) + state <- readLocal ctx name + writeLocal ctx name $ removeProperties props state + PrefixScope -> do + -- Prefix values become local + state <- readLocal ctx name + writeLocal ctx name $ removeProperties props state + + CFUndefineVariable name -> undefineVariable ctx name CFUndefineFunction name -> undefineFunction ctx name CFUndefine name -> do @@ -880,11 +1063,9 @@ transferEffect ctx effect = CFUndefineNameref name -> undefineVariable ctx name CFHintArray name -> return () CFHintDefined name -> return () - CFModifyProps {} -> return () -- _ -> error $ "Unknown effect " ++ show effect - -- Transfer the CFG's idea of a value into our VariableState cfValueToVariableValue ctx val = case val of @@ -905,12 +1086,17 @@ computeValue ctx part = CFStringLiteral str -> return $ literalToVariableValue str CFStringInteger -> return unknownIntegerValue CFStringUnknown -> return unknownVariableValue - CFStringVariable name -> readVariable ctx name + CFStringVariable name -> variableStateToValue <$> readVariable ctx name + where + variableStateToValue state = + case () of + _ | all (CFVPInteger `S.member`) $ variableProperties state -> unknownIntegerValue + _ -> variableValue state -- Append two VariableValues as if with z="$x$y" appendVariableValue :: VariableValue -> VariableValue -> VariableValue appendVariableValue a b = - VariableValue { + unknownVariableValue { literalValue = liftM2 (++) (literalValue a) (literalValue b), spaceStatus = appendSpaceStatus (spaceStatus a) (spaceStatus b) } @@ -922,12 +1108,12 @@ appendSpaceStatus a b = (SpaceStatusClean, SpaceStatusClean) -> a _ ->SpaceStatusDirty -unknownIntegerValue = VariableValue { +unknownIntegerValue = unknownVariableValue { literalValue = Nothing, spaceStatus = SpaceStatusClean } -literalToVariableValue str = VariableValue { +literalToVariableValue str = unknownVariableValue { literalValue = Just str, spaceStatus = literalToSpaceStatus str } @@ -965,6 +1151,13 @@ dataflow ctx entry = do f 0 _ _ = error $ pleaseReport "DFA did not reach fix point" f n pending states = do ps <- readSTRef pending + + when (n == fallbackThreshold) $ do + -- This should never happen, but has historically been due to caching bugs. + -- Try disabling the cache and continuing. + logInfo "DFA is not stabilizing! Disabling cache." + writeSTRef (cEnableCache ctx) False + if S.null ps then return () else do @@ -1012,12 +1205,12 @@ runRoot ctx entry exit = do writeSTRef (cInput ctx) $ env writeSTRef (cOutput ctx) $ env writeSTRef (cNode ctx) $ entry - (states, frame) <- withNewStackFrame ctx entry $ \c -> dataflow c entry + (states, frame) <- withNewStackFrame ctx entry False $ \c -> dataflow c entry deps <- readSTRef $ dependencies frame registerFlowResult ctx entry states deps -- Return the final state, used to invoke functions that were declared but not invoked return $ snd $ fromMaybe (error $ pleaseReport "Missing exit state") $ M.lookup exit states - + analyzeControlFlow :: CFGParameters -> Token -> CFGAnalysis analyzeControlFlow params t = From e7f05d662ad1830cc9b4bce59a7b6767fa07d8c0 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Fri, 22 Jul 2022 10:29:19 -0700 Subject: [PATCH 243/459] In addition to start/end, track sets of nodes belonging to tokens --- src/ShellCheck/CFG.hs | 66 ++++++++++++++++++++++------------- src/ShellCheck/CFGAnalysis.hs | 12 ++++--- src/ShellCheck/Debug.hs | 2 +- 3 files changed, 50 insertions(+), 30 deletions(-) diff --git a/src/ShellCheck/CFG.hs b/src/ShellCheck/CFG.hs index a4bd166..4906d80 100644 --- a/src/ShellCheck/CFG.hs +++ b/src/ShellCheck/CFG.hs @@ -168,8 +168,10 @@ data CFGParameters = CFGParameters { data CFGResult = CFGResult { -- The graph itself cfGraph :: CFGraph, - -- Map from Id to start/end node - cfIdToNode :: M.Map Id (Node, Node) + -- Map from Id to nominal start&end node (i.e. assuming normal execution without exits) + cfIdToRange :: M.Map Id (Node, Node), + -- A set of all nodes belonging to an Id, recursively + cfIdToNodes :: M.Map Id (S.Set Node) } deriving (Show) @@ -177,21 +179,24 @@ buildGraph :: CFGParameters -> Token -> CFGResult buildGraph params root = let (nextNode, base) = execRWS (buildRoot root) (newCFContext params) 0 - (nodes, edges, mapping) = + (nodes, edges, mapping, association) = -- renumberTopologically $ removeUnnecessaryStructuralNodes base in CFGResult { cfGraph = mkGraph nodes edges, - cfIdToNode = M.fromList mapping + cfIdToRange = M.fromList mapping, + cfIdToNodes = M.fromListWith S.union $ map (\(id, n) -> (id, S.singleton n)) association } -remapGraph remap (nodes, edges, mapping) = +remapGraph :: M.Map Node Node -> CFW -> CFW +remapGraph remap (nodes, edges, mapping, assoc) = ( map (remapNode remap) nodes, map (remapEdge remap) edges, - map (\(id, (a,b)) -> (id, (remapHelper remap a, remapHelper remap b))) mapping + map (\(id, (a,b)) -> (id, (remapHelper remap a, remapHelper remap b))) mapping, + map (\(id, n) -> (id, remapHelper remap n)) assoc ) prop_testRenumbering = @@ -200,17 +205,20 @@ prop_testRenumbering = before = ( [(1,s), (3,s), (4, s), (8,s)], [(1,3,CFEFlow), (3,4, CFEFlow), (4,8,CFEFlow)], - [(Id 0, (3,4))] + [(Id 0, (3,4))], + [(Id 1, 3), (Id 2, 4)] ) after = ( [(0,s), (1,s), (2,s), (3,s)], [(0,1,CFEFlow), (1,2, CFEFlow), (2,3,CFEFlow)], - [(Id 0, (1,2))] + [(Id 0, (1,2))], + [(Id 1, 1), (Id 2, 2)] ) in after == renumberGraph before -- Renumber the graph for prettiness, so there are no gaps in node numbers -renumberGraph g@(nodes, edges, mapping) = +renumberGraph :: CFW -> CFW +renumberGraph g@(nodes, edges, mapping, assoc) = let renumbering = M.fromList (flip zip [0..] $ sort $ map fst nodes) in remapGraph renumbering g @@ -220,17 +228,19 @@ prop_testRenumberTopologically = before = ( [(4,s), (2,s), (3, s)], [(4,2,CFEFlow), (2,3, CFEFlow)], - [(Id 0, (4,2))] + [(Id 0, (4,2))], + [] ) after = ( [(0,s), (1,s), (2,s)], [(0,1,CFEFlow), (1,2, CFEFlow)], - [(Id 0, (0,1))] + [(Id 0, (0,1))], + [] ) in after == renumberTopologically before -- Renumber the graph in topological order -renumberTopologically g@(nodes, edges, mapping) = +renumberTopologically g@(nodes, edges, mapping, assoc) = let renumbering = M.fromList (flip zip [0..] $ topsort (mkGraph nodes edges :: CFGraph)) in remapGraph renumbering g @@ -240,12 +250,14 @@ prop_testRemoveStructural = before = ( [(1,s), (2,s), (3, s), (4,s)], [(1,2,CFEFlow), (2,3, CFEFlow), (3,4,CFEFlow)], - [(Id 0, (2,3))] + [(Id 0, (2,3))], + [(Id 0, 3)] ) after = ( [(1,s), (2,s), (4,s)], [(1,2,CFEFlow), (2,4,CFEFlow)], - [(Id 0, (2,2))] + [(Id 0, (2,2))], + [(Id 0, 2)] ) in after == removeUnnecessaryStructuralNodes before @@ -255,12 +267,13 @@ prop_testRemoveStructural = -- Note in particular that we can't remove a structural node x in -- foo -> x -> bar , because then the pre/post-condition for tokens -- previously pointing to x would be wrong. -removeUnnecessaryStructuralNodes (nodes, edges, mapping) = +removeUnnecessaryStructuralNodes (nodes, edges, mapping, association) = remapGraph recursiveRemapping ( filter (\(n, _) -> n `M.notMember` recursiveRemapping) nodes, filter (`S.notMember` edgesToCollapse) edges, - mapping + mapping, + association ) where regularEdges = filter isRegularEdge edges @@ -305,8 +318,6 @@ remapNode m (node, label) = newLabel = case label of CFApplyEffects effects -> CFApplyEffects (map (remapEffect m) effects) CFExecuteSubshell s a b -> CFExecuteSubshell s (remapHelper m a) (remapHelper m b) --- CFSubShellStart reason node -> CFSubShellStart reason (remapHelper m node) - _ -> label remapEffect map old@(IdTagged id effect) = @@ -325,6 +336,7 @@ data CFContext = CFContext { cfIsCondition :: Bool, cfIsFunction :: Bool, cfLoopStack :: [(Node, Node)], + cfTokenStack :: [Id], cfExitTarget :: Maybe Node, cfReturnTarget :: Maybe Node, cfParameters :: CFGParameters @@ -333,19 +345,22 @@ newCFContext params = CFContext { cfIsCondition = False, cfIsFunction = False, cfLoopStack = [], + cfTokenStack = [], cfExitTarget = Nothing, cfReturnTarget = Nothing, cfParameters = params } -- The monad we generate a graph in -type CFM a = RWS CFContext ([LNode CFNode], [LEdge CFEdge], [(Id, (Node, Node))]) Int a +type CFM a = RWS CFContext CFW Int a +type CFW = ([LNode CFNode], [LEdge CFEdge], [(Id, (Node, Node))], [(Id, Node)]) newNode :: CFNode -> CFM Node newNode label = do n <- get + stack <- asks cfTokenStack put (n+1) - tell ([(n, label)], [], []) + tell ([(n, label)], [], [], map (\c -> (c, n)) stack) return n newNodeRange :: CFNode -> CFM Range @@ -367,16 +382,19 @@ withFunctionScope p = do body <- local (\c -> c { cfReturnTarget = Just end, cfIsFunction = True }) p linkRanges [body, nodeToRange end] +-- Anything that happens recursively in f will be attributed to this id +under :: Id -> CFM a -> CFM a +under id f = local (\c -> c { cfTokenStack = id:(cfTokenStack c) }) f nodeToRange :: Node -> Range nodeToRange n = Range n n link :: Node -> Node -> CFEdge -> CFM () link from to label = do - tell ([], [(from, to, label)], []) + tell ([], [(from, to, label)], [], []) registerNode :: Id -> Range -> CFM () -registerNode id (Range start end) = tell ([], [], [(id, (start, end))]) +registerNode id (Range start end) = tell ([], [], [(id, (start, end))], []) linkRange :: Range -> Range -> CFM Range linkRange = linkRangeAs CFEFlow @@ -412,7 +430,7 @@ asCondition = withContext (\c -> c { cfIsCondition = True }) newStructuralNode = newNodeRange CFStructuralNode buildRoot :: Token -> CFM Range -buildRoot t = do +buildRoot t = under (getId t) $ do entry <- newNodeRange $ CFEntryPoint "MAIN" impliedExit <- newNode CFImpliedExit end <- newNode CFStructuralNode @@ -426,7 +444,7 @@ applySingle e = CFApplyEffects [e] -- Build the CFG. build :: Token -> CFM Range build t = do - range <- build' t + range <- under (getId t) $ build' t registerNode (getId t) range return range where diff --git a/src/ShellCheck/CFGAnalysis.hs b/src/ShellCheck/CFGAnalysis.hs index 0007a67..daade43 100644 --- a/src/ShellCheck/CFGAnalysis.hs +++ b/src/ShellCheck/CFGAnalysis.hs @@ -97,7 +97,8 @@ logInfo log = do -- The result of the data flow analysis data CFGAnalysis = CFGAnalysis { graph :: CFGraph, - tokenToNode :: M.Map Id (Node, Node), + tokenToRange :: M.Map Id (Node, Node), + tokenToNodes :: M.Map Id (S.Set Node), nodeToData :: M.Map Node (ProgramState, ProgramState) } deriving (Show, Generic, NFData) @@ -111,13 +112,13 @@ data ProgramState = ProgramState { -- Conveniently get the state before a token id getIncomingState :: CFGAnalysis -> Id -> Maybe ProgramState getIncomingState analysis id = do - (start,end) <- M.lookup id $ tokenToNode analysis + (start,end) <- M.lookup id $ tokenToRange analysis fst <$> M.lookup start (nodeToData analysis) -- Conveniently get the state after a token id getOutgoingState :: CFGAnalysis -> Id -> Maybe ProgramState getOutgoingState analysis id = do - (start,end) <- M.lookup id $ tokenToNode analysis + (start,end) <- M.lookup id $ tokenToRange analysis snd <$> M.lookup end (nodeToData analysis) getDataForNode analysis node = M.lookup node $ nodeToData analysis @@ -1216,7 +1217,7 @@ analyzeControlFlow :: CFGParameters -> Token -> CFGAnalysis analyzeControlFlow params t = let cfg = buildGraph params t - (entry, exit) = M.findWithDefault (error $ pleaseReport "Missing root") (getId t) (cfIdToNode cfg) + (entry, exit) = M.findWithDefault (error $ pleaseReport "Missing root") (getId t) (cfIdToRange cfg) in runST $ f cfg entry exit where @@ -1250,7 +1251,8 @@ analyzeControlFlow params t = return $ nodeToData `deepseq` CFGAnalysis { graph = cfGraph cfg, - tokenToNode = cfIdToNode cfg, + tokenToRange = cfIdToRange cfg, + tokenToNodes = cfIdToNodes cfg, nodeToData = nodeToData } diff --git a/src/ShellCheck/Debug.hs b/src/ShellCheck/Debug.hs index c991308..b6015e5 100644 --- a/src/ShellCheck/Debug.hs +++ b/src/ShellCheck/Debug.hs @@ -202,7 +202,7 @@ stringToDetailedCfgViz scriptString = cfgToGraphVizWith nodeLabel graph idToToken = M.fromList $ execWriter $ doAnalysis (\c -> tell [(getId c, c)]) ast idToNode :: M.Map Id (Node, Node) - idToNode = cfIdToNode cfgResult + idToNode = cfIdToRange cfgResult nodeToStartIds :: M.Map Node (S.Set Id) nodeToStartIds = From 95b3cbf0714df47feba9ff8d465c27cb6503f38f Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Fri, 22 Jul 2022 11:11:09 -0700 Subject: [PATCH 244/459] Qualify Data.Map as M instead of tedious Map --- src/ShellCheck/Checks/Commands.hs | 42 +++++++++++++++---------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index cac06bc..081ec5f 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -39,7 +39,7 @@ import Data.Char import Data.Functor.Identity import Data.List import Data.Maybe -import qualified Data.Map.Strict as Map +import qualified Data.Map.Strict as M import Test.QuickCheck.All (forAllProperties) import Test.QuickCheck.Test (quickCheckWithResult, stdArgs, maxSuccess) @@ -114,7 +114,7 @@ optionalCommandChecks = [ cdNegative = "command -v javac" }, checkWhich) ] -optionalCheckMap = Map.fromList $ map (\(desc, check) -> (cdName desc, check)) optionalCommandChecks +optionalCheckMap = M.fromList $ map (\(desc, check) -> (cdName desc, check)) optionalCommandChecks prop_verifyOptionalExamples = all check optionalCommandChecks where @@ -163,27 +163,27 @@ prop_checkGenericOptsT1 = checkGetOpts "-x -- -y" ["x"] ["-y"] $ return . getGen prop_checkGenericOptsT2 = checkGetOpts "-xy --" ["x", "y"] [] $ return . getGenericOpts -buildCommandMap :: [CommandCheck] -> Map.Map CommandName (Token -> Analysis) -buildCommandMap = foldl' addCheck Map.empty +buildCommandMap :: [CommandCheck] -> M.Map CommandName (Token -> Analysis) +buildCommandMap = foldl' addCheck M.empty where addCheck map (CommandCheck name function) = - Map.insertWith composeAnalyzers name function map + M.insertWith composeAnalyzers name function map -checkCommand :: Map.Map CommandName (Token -> Analysis) -> Token -> Analysis +checkCommand :: M.Map CommandName (Token -> Analysis) -> Token -> Analysis checkCommand map t@(T_SimpleCommand id cmdPrefix (cmd:rest)) = sequence_ $ do name <- getLiteralString cmd return $ if '/' `elem` name then - Map.findWithDefault nullCheck (Basename $ basename name) map t + M.findWithDefault nullCheck (Basename $ basename name) map t else if name == "builtin" && not (null rest) then let t' = T_SimpleCommand id cmdPrefix rest selectedBuiltin = fromMaybe "" $ getLiteralString . head $ rest - in Map.findWithDefault nullCheck (Exactly selectedBuiltin) map t' + in M.findWithDefault nullCheck (Exactly selectedBuiltin) map t' else do - Map.findWithDefault nullCheck (Exactly name) map t - Map.findWithDefault nullCheck (Basename name) map t + M.findWithDefault nullCheck (Exactly name) map t + M.findWithDefault nullCheck (Basename name) map t where basename = reverse . takeWhile (/= '/') . reverse @@ -205,7 +205,7 @@ checker spec params = getChecker $ commandChecks ++ optionals optionals = if "all" `elem` keys then map snd optionalCommandChecks - else mapMaybe (\x -> Map.lookup x optionalCheckMap) keys + else mapMaybe (\x -> M.lookup x optionalCheckMap) keys prop_checkTr1 = verify checkTr "tr [a-f] [A-F]" prop_checkTr2 = verify checkTr "tr 'a-z' 'A-Z'" @@ -1005,20 +1005,20 @@ checkWhileGetoptsCase = CommandCheck (Exactly "getopts") f check :: Id -> [String] -> Token -> Analysis check optId opts (T_CaseExpression id _ list) = do - unless (Nothing `Map.member` handledMap) $ do - mapM_ (warnUnhandled optId id) $ catMaybes $ Map.keys notHandled + unless (Nothing `M.member` handledMap) $ do + mapM_ (warnUnhandled optId id) $ catMaybes $ M.keys notHandled - unless (any (`Map.member` handledMap) [Just "*",Just "?"]) $ + unless (any (`M.member` handledMap) [Just "*",Just "?"]) $ warn id 2220 "Invalid flags are not handled. Add a *) case." - mapM_ warnRedundant $ Map.toList notRequested + mapM_ warnRedundant $ M.toList notRequested where - handledMap = Map.fromList (concatMap getHandledStrings list) - requestedMap = Map.fromList $ map (\x -> (Just x, ())) opts + handledMap = M.fromList (concatMap getHandledStrings list) + requestedMap = M.fromList $ map (\x -> (Just x, ())) opts - notHandled = Map.difference requestedMap handledMap - notRequested = Map.difference handledMap requestedMap + notHandled = M.difference requestedMap handledMap + notRequested = M.difference handledMap requestedMap warnUnhandled optId caseId str = warn caseId 2213 $ "getopts specified -" ++ (e4m str) ++ ", but it's not handled by this 'case'." @@ -1372,10 +1372,10 @@ checkUnquotedEchoSpaces = CommandCheck (Basename "echo") check m <- asks tokenPositions redir <- getClosestCommandM t sequence_ $ do - let positions = mapMaybe (\c -> Map.lookup (getId c) m) args + let positions = mapMaybe (\c -> M.lookup (getId c) m) args let pairs = zip positions (drop 1 positions) (T_Redirecting _ redirTokens _) <- redir - let redirPositions = mapMaybe (\c -> fst <$> Map.lookup (getId c) m) redirTokens + let redirPositions = mapMaybe (\c -> fst <$> M.lookup (getId c) m) redirTokens guard $ any (hasSpacesBetween redirPositions) pairs return $ info (getId t) 2291 "Quote repeated spaces to avoid them collapsing into one." From 9caeec104b5b70054c1b4238d4dfde4c641ea12e Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Fri, 22 Jul 2022 11:25:07 -0700 Subject: [PATCH 245/459] SC2318: Warn about backreferencing in `declare x=1 y=$x` (fixes #1653) --- CHANGELOG.md | 1 + src/ShellCheck/Checks/Commands.hs | 53 +++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7da6077..dce3e27 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ### Added - SC2316: Warn about 'local readonly foo' and similar (thanks, patrickxia!) - SC2317: Warn about unreachable commands +- SC2318: Warn about backreferences in 'declare x=1 y=$x' ### Fixed - SC2086: Now uses DFA to make more accurate predictions about values diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 081ec5f..acbf967 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -27,6 +27,8 @@ module ShellCheck.Checks.Commands (checker, optionalChecks, ShellCheck.Checks.Co import ShellCheck.AST import ShellCheck.ASTLib import ShellCheck.AnalyzerLib +import ShellCheck.CFG +import qualified ShellCheck.CFGAnalysis as CF import ShellCheck.Data import ShellCheck.Interface import ShellCheck.Parser @@ -37,12 +39,16 @@ import Control.Monad import Control.Monad.RWS import Data.Char import Data.Functor.Identity +import qualified Data.Graph.Inductive.Graph as G import Data.List import Data.Maybe import qualified Data.Map.Strict as M +import qualified Data.Set as S import Test.QuickCheck.All (forAllProperties) import Test.QuickCheck.Test (quickCheckWithResult, stdArgs, maxSuccess) +import Debug.Trace -- STRIP + data CommandName = Exactly String | Basename String deriving (Eq, Ord) @@ -102,6 +108,7 @@ commandChecks = [ ++ map checkArgComparison ("alias" : declaringCommands) ++ map checkMaskedReturns declaringCommands ++ map checkMultipleDeclaring declaringCommands + ++ map checkBackreferencingDeclaration declaringCommands optionalChecks = map fst optionalCommandChecks @@ -1405,5 +1412,51 @@ checkEvalArray = CommandCheck (Exactly "eval") (mapM_ check . concatMap getWordP _ -> False +prop_checkBackreferencingDeclaration1 = verify (checkBackreferencingDeclaration "declare") "declare x=1 y=foo$x" +prop_checkBackreferencingDeclaration2 = verify (checkBackreferencingDeclaration "readonly") "readonly x=1 y=$((1+x))" +prop_checkBackreferencingDeclaration3 = verify (checkBackreferencingDeclaration "local") "local x=1 y=$(echo $x)" +prop_checkBackreferencingDeclaration4 = verify (checkBackreferencingDeclaration "local") "local x=1 y[$x]=z" +prop_checkBackreferencingDeclaration5 = verify (checkBackreferencingDeclaration "declare") "declare x=var $x=1" +prop_checkBackreferencingDeclaration6 = verify (checkBackreferencingDeclaration "declare") "declare x=var $x=1" +prop_checkBackreferencingDeclaration7 = verify (checkBackreferencingDeclaration "declare") "declare x=var $k=$x" +checkBackreferencingDeclaration cmd = CommandCheck (Exactly cmd) check + where + check t = foldM_ perArg M.empty $ arguments t + + perArg leftArgs t = + case t of + T_Assignment id _ name idx t -> do + warnIfBackreferencing leftArgs $ t:idx + return $ M.insert name id leftArgs + t -> do + warnIfBackreferencing leftArgs [t] + return leftArgs + + warnIfBackreferencing backrefs l = do + references <- findReferences l + let reused = M.intersection backrefs references + mapM msg $ M.toList reused + + msg (name, id) = warn id 2318 $ "This assignment is used again in this '" ++ cmd ++ "', but won't have taken effect. Use two '" ++ cmd ++ "'s." + + findReferences list = do + cfga <- asks cfgAnalysis + let graph = CF.graph cfga + let nodesMap = CF.tokenToNodes cfga + let nodes = S.unions $ map (\id -> M.findWithDefault S.empty id nodesMap) $ map getId $ list + let labels = mapMaybe (G.lab graph) $ S.toList nodes + let references = M.fromList $ concatMap refFromLabel labels + return references + + refFromLabel lab = + case lab of + CFApplyEffects effects -> mapMaybe refFromEffect effects + _ -> [] + refFromEffect e = + case e of + IdTagged id (CFReadVariable name) -> return (name, id) + _ -> Nothing + + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) From e47480e93af07a7e72ee0eb8c0d7d57c92ec35e4 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Fri, 22 Jul 2022 16:28:24 -0700 Subject: [PATCH 246/459] Also emit SC2004 for array indices (fixes #1666) --- src/ShellCheck/Analytics.hs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index ca918ba..8b8d489 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1460,7 +1460,8 @@ prop_checkArithmeticDeref7 = verifyNot checkArithmeticDeref "(( 10#$n ))" prop_checkArithmeticDeref8 = verifyNot checkArithmeticDeref "let i=$i+1" prop_checkArithmeticDeref9 = verifyNot checkArithmeticDeref "(( a[foo] ))" prop_checkArithmeticDeref10= verifyNot checkArithmeticDeref "(( a[\\$foo] ))" -prop_checkArithmeticDeref11= verifyNot checkArithmeticDeref "a[$foo]=wee" +prop_checkArithmeticDeref11= verify checkArithmeticDeref "a[$foo]=wee" +prop_checkArithmeticDeref11b= verifyNot checkArithmeticDeref "declare -A a; a[$foo]=wee" prop_checkArithmeticDeref12= verify checkArithmeticDeref "for ((i=0; $i < 3; i)); do true; done" prop_checkArithmeticDeref13= verifyNot checkArithmeticDeref "(( $$ ))" prop_checkArithmeticDeref14= verifyNot checkArithmeticDeref "(( $! ))" @@ -1477,6 +1478,7 @@ checkArithmeticDeref params t@(TA_Expansion _ [T_DollarBraced id _ l]) = T_Arithmetic {} -> return normalWarning T_DollarArithmetic {} -> return normalWarning T_ForArithmetic {} -> return normalWarning + T_Assignment {} -> return normalWarning T_SimpleCommand {} -> return noWarning _ -> Nothing From 2f28847b0897a8f91c6f227ed098470eea383dd4 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Fri, 22 Jul 2022 16:35:14 -0700 Subject: [PATCH 247/459] Normalize spaces around = in unit tests --- src/ShellCheck/ASTLib.hs | 10 +- src/ShellCheck/Analytics.hs | 392 +++++++++++++------------- src/ShellCheck/Checks/Commands.hs | 76 ++--- src/ShellCheck/Checks/ShellSupport.hs | 106 +++---- src/ShellCheck/Parser.hs | 90 +++--- 5 files changed, 337 insertions(+), 337 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 7cc5af2..7b4f9e5 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -810,11 +810,11 @@ prop_getBracedReference6 = getBracedReference "!#" == "#" prop_getBracedReference7 = getBracedReference "!foo#?" == "foo" prop_getBracedReference8 = getBracedReference "foo-bar" == "foo" prop_getBracedReference9 = getBracedReference "foo:-bar" == "foo" -prop_getBracedReference10= getBracedReference "foo: -1" == "foo" -prop_getBracedReference11= getBracedReference "!os*" == "" -prop_getBracedReference11b= getBracedReference "!os@" == "" -prop_getBracedReference12= getBracedReference "!os?bar**" == "" -prop_getBracedReference13= getBracedReference "foo[bar]" == "foo" +prop_getBracedReference10 = getBracedReference "foo: -1" == "foo" +prop_getBracedReference11 = getBracedReference "!os*" == "" +prop_getBracedReference11b = getBracedReference "!os@" == "" +prop_getBracedReference12 = getBracedReference "!os?bar**" == "" +prop_getBracedReference13 = getBracedReference "foo[bar]" == "foo" getBracedReference s = fromMaybe s $ nameExpansion s `mplus` takeName noPrefix `mplus` getSpecial noPrefix `mplus` getSpecial s where diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 8b8d489..1429e1b 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -638,15 +638,15 @@ prop_checkShebang6 = verifyNotTree checkShebang "#!/usr/bin/env ash\n# shellchec prop_checkShebang7 = verifyNotTree checkShebang "#!/usr/bin/env ash\n# shellcheck shell=sh\n" prop_checkShebang8 = verifyTree checkShebang "#!bin/sh\ntrue" prop_checkShebang9 = verifyNotTree checkShebang "# shellcheck shell=sh\ntrue" -prop_checkShebang10= verifyNotTree checkShebang "#!foo\n# shellcheck shell=sh ignore=SC2239\ntrue" -prop_checkShebang11= verifyTree checkShebang "#!/bin/sh/\ntrue" -prop_checkShebang12= verifyTree checkShebang "#!/bin/sh/ -xe\ntrue" -prop_checkShebang13= verifyTree checkShebang "#!/bin/busybox sh" -prop_checkShebang14= verifyNotTree checkShebang "#!/bin/busybox sh\n# shellcheck shell=sh\n" -prop_checkShebang15= verifyNotTree checkShebang "#!/bin/busybox sh\n# shellcheck shell=dash\n" -prop_checkShebang16= verifyTree checkShebang "#!/bin/busybox ash" -prop_checkShebang17= verifyNotTree checkShebang "#!/bin/busybox ash\n# shellcheck shell=dash\n" -prop_checkShebang18= verifyNotTree checkShebang "#!/bin/busybox ash\n# shellcheck shell=sh\n" +prop_checkShebang10 = verifyNotTree checkShebang "#!foo\n# shellcheck shell=sh ignore=SC2239\ntrue" +prop_checkShebang11 = verifyTree checkShebang "#!/bin/sh/\ntrue" +prop_checkShebang12 = verifyTree checkShebang "#!/bin/sh/ -xe\ntrue" +prop_checkShebang13 = verifyTree checkShebang "#!/bin/busybox sh" +prop_checkShebang14 = verifyNotTree checkShebang "#!/bin/busybox sh\n# shellcheck shell=sh\n" +prop_checkShebang15 = verifyNotTree checkShebang "#!/bin/busybox sh\n# shellcheck shell=dash\n" +prop_checkShebang16 = verifyTree checkShebang "#!/bin/busybox ash" +prop_checkShebang17 = verifyNotTree checkShebang "#!/bin/busybox ash\n# shellcheck shell=dash\n" +prop_checkShebang18 = verifyNotTree checkShebang "#!/bin/busybox ash\n# shellcheck shell=sh\n" checkShebang params (T_Annotation _ list t) = if any isOverride list then [] else checkShebang params t where @@ -700,9 +700,9 @@ checkForInQuoted params (T_ForIn _ _ multiple _) = checkForInQuoted _ _ = return () prop_checkForInCat1 = verify checkForInCat "for f in $(cat foo); do stuff; done" -prop_checkForInCat1a= verify checkForInCat "for f in `cat foo`; do stuff; done" +prop_checkForInCat1a = verify checkForInCat "for f in `cat foo`; do stuff; done" prop_checkForInCat2 = verify checkForInCat "for f in $(cat foo | grep lol); do stuff; done" -prop_checkForInCat2a= verify checkForInCat "for f in `cat foo | grep lol`; do stuff; done" +prop_checkForInCat2a = verify checkForInCat "for f in `cat foo | grep lol`; do stuff; done" prop_checkForInCat3 = verifyNot checkForInCat "for f in $(cat foo | grep bar | wc -l); do stuff; done" checkForInCat _ (T_ForIn _ f [T_NormalWord _ w] _) = mapM_ checkF w where @@ -775,10 +775,10 @@ checkFindExec _ _ = return () prop_checkUnquotedExpansions1 = verify checkUnquotedExpansions "rm $(ls)" -prop_checkUnquotedExpansions1a= verify checkUnquotedExpansions "rm `ls`" +prop_checkUnquotedExpansions1a = verify checkUnquotedExpansions "rm `ls`" prop_checkUnquotedExpansions2 = verify checkUnquotedExpansions "rm foo$(date)" prop_checkUnquotedExpansions3 = verify checkUnquotedExpansions "[ $(foo) == cow ]" -prop_checkUnquotedExpansions3a= verify checkUnquotedExpansions "[ ! $(foo) ]" +prop_checkUnquotedExpansions3a = verify checkUnquotedExpansions "[ ! $(foo) ]" prop_checkUnquotedExpansions4 = verifyNot checkUnquotedExpansions "[[ $(foo) == cow ]]" prop_checkUnquotedExpansions5 = verifyNot checkUnquotedExpansions "for f in $(cmd); do echo $f; done" prop_checkUnquotedExpansions6 = verifyNot checkUnquotedExpansions "$(cmd)" @@ -906,7 +906,7 @@ checkDollarStar _ _ = return () prop_checkUnquotedDollarAt = verify checkUnquotedDollarAt "ls $@" -prop_checkUnquotedDollarAt1= verifyNot checkUnquotedDollarAt "ls ${#@}" +prop_checkUnquotedDollarAt1 = verifyNot checkUnquotedDollarAt "ls ${#@}" prop_checkUnquotedDollarAt2 = verify checkUnquotedDollarAt "ls ${foo[@]}" prop_checkUnquotedDollarAt3 = verifyNot checkUnquotedDollarAt "ls ${#foo[@]}" prop_checkUnquotedDollarAt4 = verifyNot checkUnquotedDollarAt "ls \"$@\"" @@ -1037,32 +1037,32 @@ ltt t = trace ("Tracing " ++ show t) -- STRIP prop_checkSingleQuotedVariables = verify checkSingleQuotedVariables "echo '$foo'" prop_checkSingleQuotedVariables2 = verify checkSingleQuotedVariables "echo 'lol$1.jpg'" prop_checkSingleQuotedVariables3 = verifyNot checkSingleQuotedVariables "sed 's/foo$/bar/'" -prop_checkSingleQuotedVariables3a= verify checkSingleQuotedVariables "sed 's/${foo}/bar/'" -prop_checkSingleQuotedVariables3b= verify checkSingleQuotedVariables "sed 's/$(echo cow)/bar/'" -prop_checkSingleQuotedVariables3c= verify checkSingleQuotedVariables "sed 's/$((1+foo))/bar/'" +prop_checkSingleQuotedVariables3a = verify checkSingleQuotedVariables "sed 's/${foo}/bar/'" +prop_checkSingleQuotedVariables3b = verify checkSingleQuotedVariables "sed 's/$(echo cow)/bar/'" +prop_checkSingleQuotedVariables3c = verify checkSingleQuotedVariables "sed 's/$((1+foo))/bar/'" prop_checkSingleQuotedVariables4 = verifyNot checkSingleQuotedVariables "awk '{print $1}'" prop_checkSingleQuotedVariables5 = verifyNot checkSingleQuotedVariables "trap 'echo $SECONDS' EXIT" prop_checkSingleQuotedVariables6 = verifyNot checkSingleQuotedVariables "sed -n '$p'" -prop_checkSingleQuotedVariables6a= verify checkSingleQuotedVariables "sed -n '$pattern'" +prop_checkSingleQuotedVariables6a = verify checkSingleQuotedVariables "sed -n '$pattern'" prop_checkSingleQuotedVariables7 = verifyNot checkSingleQuotedVariables "PS1='$PWD \\$ '" prop_checkSingleQuotedVariables8 = verify checkSingleQuotedVariables "find . -exec echo '$1' {} +" prop_checkSingleQuotedVariables9 = verifyNot checkSingleQuotedVariables "find . -exec awk '{print $1}' {} \\;" -prop_checkSingleQuotedVariables10= verify checkSingleQuotedVariables "echo '`pwd`'" -prop_checkSingleQuotedVariables11= verifyNot checkSingleQuotedVariables "sed '${/lol/d}'" -prop_checkSingleQuotedVariables12= verifyNot checkSingleQuotedVariables "eval 'echo $1'" -prop_checkSingleQuotedVariables13= verifyNot checkSingleQuotedVariables "busybox awk '{print $1}'" -prop_checkSingleQuotedVariables14= verifyNot checkSingleQuotedVariables "[ -v 'bar[$foo]' ]" -prop_checkSingleQuotedVariables15= verifyNot checkSingleQuotedVariables "git filter-branch 'test $GIT_COMMIT'" -prop_checkSingleQuotedVariables16= verify checkSingleQuotedVariables "git '$a'" -prop_checkSingleQuotedVariables17= verifyNot checkSingleQuotedVariables "rename 's/(.)a/$1/g' *" -prop_checkSingleQuotedVariables18= verifyNot checkSingleQuotedVariables "echo '``'" -prop_checkSingleQuotedVariables19= verifyNot checkSingleQuotedVariables "echo '```'" -prop_checkSingleQuotedVariables20= verifyNot checkSingleQuotedVariables "mumps -run %XCMD 'W $O(^GLOBAL(5))'" -prop_checkSingleQuotedVariables21= verifyNot checkSingleQuotedVariables "mumps -run LOOP%XCMD --xec 'W $O(^GLOBAL(6))'" -prop_checkSingleQuotedVariables22= verifyNot checkSingleQuotedVariables "jq '$__loc__'" -prop_checkSingleQuotedVariables23= verifyNot checkSingleQuotedVariables "command jq '$__loc__'" -prop_checkSingleQuotedVariables24= verifyNot checkSingleQuotedVariables "exec jq '$__loc__'" -prop_checkSingleQuotedVariables25= verifyNot checkSingleQuotedVariables "exec -c -a foo jq '$__loc__'" +prop_checkSingleQuotedVariables10 = verify checkSingleQuotedVariables "echo '`pwd`'" +prop_checkSingleQuotedVariables11 = verifyNot checkSingleQuotedVariables "sed '${/lol/d}'" +prop_checkSingleQuotedVariables12 = verifyNot checkSingleQuotedVariables "eval 'echo $1'" +prop_checkSingleQuotedVariables13 = verifyNot checkSingleQuotedVariables "busybox awk '{print $1}'" +prop_checkSingleQuotedVariables14 = verifyNot checkSingleQuotedVariables "[ -v 'bar[$foo]' ]" +prop_checkSingleQuotedVariables15 = verifyNot checkSingleQuotedVariables "git filter-branch 'test $GIT_COMMIT'" +prop_checkSingleQuotedVariables16 = verify checkSingleQuotedVariables "git '$a'" +prop_checkSingleQuotedVariables17 = verifyNot checkSingleQuotedVariables "rename 's/(.)a/$1/g' *" +prop_checkSingleQuotedVariables18 = verifyNot checkSingleQuotedVariables "echo '``'" +prop_checkSingleQuotedVariables19 = verifyNot checkSingleQuotedVariables "echo '```'" +prop_checkSingleQuotedVariables20 = verifyNot checkSingleQuotedVariables "mumps -run %XCMD 'W $O(^GLOBAL(5))'" +prop_checkSingleQuotedVariables21 = verifyNot checkSingleQuotedVariables "mumps -run LOOP%XCMD --xec 'W $O(^GLOBAL(6))'" +prop_checkSingleQuotedVariables22 = verifyNot checkSingleQuotedVariables "jq '$__loc__'" +prop_checkSingleQuotedVariables23 = verifyNot checkSingleQuotedVariables "command jq '$__loc__'" +prop_checkSingleQuotedVariables24 = verifyNot checkSingleQuotedVariables "exec jq '$__loc__'" +prop_checkSingleQuotedVariables25 = verifyNot checkSingleQuotedVariables "exec -c -a foo jq '$__loc__'" checkSingleQuotedVariables params t@(T_SingleQuoted id s) = @@ -1353,8 +1353,8 @@ checkGlobbedRegex _ _ = return () prop_checkConstantIfs1 = verify checkConstantIfs "[[ foo != bar ]]" -prop_checkConstantIfs2a= verify checkConstantIfs "[ n -le 4 ]" -prop_checkConstantIfs2b= verifyNot checkConstantIfs "[[ n -le 4 ]]" +prop_checkConstantIfs2a = verify checkConstantIfs "[ n -le 4 ]" +prop_checkConstantIfs2b = verifyNot checkConstantIfs "[[ n -le 4 ]]" prop_checkConstantIfs3 = verify checkConstantIfs "[[ $n -le 4 && n != 2 ]]" prop_checkConstantIfs4 = verifyNot checkConstantIfs "[[ $n -le 3 ]]" prop_checkConstantIfs5 = verifyNot checkConstantIfs "[[ $n -le $n ]]" @@ -1459,14 +1459,14 @@ prop_checkArithmeticDeref6 = verify checkArithmeticDeref "(( a[$i] ))" prop_checkArithmeticDeref7 = verifyNot checkArithmeticDeref "(( 10#$n ))" prop_checkArithmeticDeref8 = verifyNot checkArithmeticDeref "let i=$i+1" prop_checkArithmeticDeref9 = verifyNot checkArithmeticDeref "(( a[foo] ))" -prop_checkArithmeticDeref10= verifyNot checkArithmeticDeref "(( a[\\$foo] ))" -prop_checkArithmeticDeref11= verify checkArithmeticDeref "a[$foo]=wee" -prop_checkArithmeticDeref11b= verifyNot checkArithmeticDeref "declare -A a; a[$foo]=wee" -prop_checkArithmeticDeref12= verify checkArithmeticDeref "for ((i=0; $i < 3; i)); do true; done" -prop_checkArithmeticDeref13= verifyNot checkArithmeticDeref "(( $$ ))" -prop_checkArithmeticDeref14= verifyNot checkArithmeticDeref "(( $! ))" -prop_checkArithmeticDeref15= verifyNot checkArithmeticDeref "(( ${!var} ))" -prop_checkArithmeticDeref16= verifyNot checkArithmeticDeref "(( ${x+1} + ${x=42} ))" +prop_checkArithmeticDeref10 = verifyNot checkArithmeticDeref "(( a[\\$foo] ))" +prop_checkArithmeticDeref11 = verify checkArithmeticDeref "a[$foo]=wee" +prop_checkArithmeticDeref11b = verifyNot checkArithmeticDeref "declare -A a; a[$foo]=wee" +prop_checkArithmeticDeref12 = verify checkArithmeticDeref "for ((i=0; $i < 3; i)); do true; done" +prop_checkArithmeticDeref13 = verifyNot checkArithmeticDeref "(( $$ ))" +prop_checkArithmeticDeref14 = verifyNot checkArithmeticDeref "(( $! ))" +prop_checkArithmeticDeref15 = verifyNot checkArithmeticDeref "(( ${!var} ))" +prop_checkArithmeticDeref16 = verifyNot checkArithmeticDeref "(( ${x+1} + ${x=42} ))" checkArithmeticDeref params t@(TA_Expansion _ [T_DollarBraced id _ l]) = unless (isException $ concat $ oversimplify l) getWarning where @@ -1602,7 +1602,7 @@ checkOrNeq _ _ = return () prop_checkValidCondOps1 = verify checkValidCondOps "[[ a -xz b ]]" prop_checkValidCondOps2 = verify checkValidCondOps "[ -M a ]" -prop_checkValidCondOps2a= verifyNot checkValidCondOps "[ 3 \\> 2 ]" +prop_checkValidCondOps2a = verifyNot checkValidCondOps "[ 3 \\> 2 ]" prop_checkValidCondOps3 = verifyNot checkValidCondOps "[ 1 = 2 -a 3 -ge 4 ]" prop_checkValidCondOps4 = verifyNot checkValidCondOps "[[ ! -v foo ]]" checkValidCondOps _ (TC_Binary id _ s _ _) @@ -1670,11 +1670,11 @@ checkTestRedirects _ (T_Redirecting id redirs cmd) | cmd `isCommand` "test" = checkTestRedirects _ _ = return () prop_checkPS11 = verify checkPS1Assignments "PS1='\\033[1;35m\\$ '" -prop_checkPS11a= verify checkPS1Assignments "export PS1='\\033[1;35m\\$ '" +prop_checkPS11a = verify checkPS1Assignments "export PS1='\\033[1;35m\\$ '" prop_checkPSf2 = verify checkPS1Assignments "PS1='\\h \\e[0m\\$ '" prop_checkPS13 = verify checkPS1Assignments "PS1=$'\\x1b[c '" prop_checkPS14 = verify checkPS1Assignments "PS1=$'\\e[3m; '" -prop_checkPS14a= verify checkPS1Assignments "export PS1=$'\\e[3m; '" +prop_checkPS14a = verify checkPS1Assignments "export PS1=$'\\e[3m; '" prop_checkPS15 = verifyNot checkPS1Assignments "PS1='\\[\\033[1;35m\\]\\$ '" prop_checkPS16 = verifyNot checkPS1Assignments "PS1='\\[\\e1m\\e[1m\\]\\$ '" prop_checkPS17 = verifyNot checkPS1Assignments "PS1='e033x1B'" @@ -1943,7 +1943,7 @@ prop_subshellAssignmentCheck3 = verifyTree subshellAssignmentCheck "( A=foo; prop_subshellAssignmentCheck4 = verifyNotTree subshellAssignmentCheck "( A=foo; rm $A; )" prop_subshellAssignmentCheck5 = verifyTree subshellAssignmentCheck "cat foo | while read cow; do true; done; echo $cow;" prop_subshellAssignmentCheck6 = verifyTree subshellAssignmentCheck "( export lol=$(ls); ); echo $lol;" -prop_subshellAssignmentCheck6a= verifyTree subshellAssignmentCheck "( typeset -a lol=a; ); echo $lol;" +prop_subshellAssignmentCheck6a = verifyTree subshellAssignmentCheck "( typeset -a lol=a; ); echo $lol;" prop_subshellAssignmentCheck7 = verifyTree subshellAssignmentCheck "cmd | while read foo; do (( n++ )); done; echo \"$n lines\"" prop_subshellAssignmentCheck8 = verifyTree subshellAssignmentCheck "n=3 & echo $((n++))" prop_subshellAssignmentCheck9 = verifyTree subshellAssignmentCheck "read n & n=foo$n" @@ -2027,67 +2027,67 @@ prop_checkSpacefulnessCfg1 = verify checkSpacefulnessCfg "a='cow moo'; echo $a" prop_checkSpacefulnessCfg2 = verifyNot checkSpacefulnessCfg "a='cow moo'; [[ $a ]]" prop_checkSpacefulnessCfg3 = verifyNot checkSpacefulnessCfg "a='cow*.mp3'; echo \"$a\"" prop_checkSpacefulnessCfg4 = verify checkSpacefulnessCfg "for f in *.mp3; do echo $f; done" -prop_checkSpacefulnessCfg4a= verifyNot checkSpacefulnessCfg "foo=3; foo=$(echo $foo)" +prop_checkSpacefulnessCfg4a = verifyNot checkSpacefulnessCfg "foo=3; foo=$(echo $foo)" prop_checkSpacefulnessCfg5 = verify checkSpacefulnessCfg "a='*'; b=$a; c=lol${b//foo/bar}; echo $c" prop_checkSpacefulnessCfg6 = verify checkSpacefulnessCfg "a=foo$(lol); echo $a" prop_checkSpacefulnessCfg7 = verify checkSpacefulnessCfg "a=foo\\ bar; rm $a" prop_checkSpacefulnessCfg8 = verifyNot checkSpacefulnessCfg "a=foo\\ bar; a=foo; rm $a" -prop_checkSpacefulnessCfg10= verify checkSpacefulnessCfg "rm $1" -prop_checkSpacefulnessCfg11= verify checkSpacefulnessCfg "rm ${10//foo/bar}" -prop_checkSpacefulnessCfg12= verifyNot checkSpacefulnessCfg "(( $1 + 3 ))" -prop_checkSpacefulnessCfg13= verifyNot checkSpacefulnessCfg "if [[ $2 -gt 14 ]]; then true; fi" -prop_checkSpacefulnessCfg14= verifyNot checkSpacefulnessCfg "foo=$3 env" -prop_checkSpacefulnessCfg15= verifyNot checkSpacefulnessCfg "local foo=$1" -prop_checkSpacefulnessCfg16= verifyNot checkSpacefulnessCfg "declare foo=$1" -prop_checkSpacefulnessCfg17= verify checkSpacefulnessCfg "echo foo=$1" -prop_checkSpacefulnessCfg18= verifyNot checkSpacefulnessCfg "$1 --flags" -prop_checkSpacefulnessCfg19= verify checkSpacefulnessCfg "echo $PWD" -prop_checkSpacefulnessCfg20= verifyNot checkSpacefulnessCfg "n+='foo bar'" -prop_checkSpacefulnessCfg21= verifyNot checkSpacefulnessCfg "select foo in $bar; do true; done" -prop_checkSpacefulnessCfg22= verifyNot checkSpacefulnessCfg "echo $\"$1\"" -prop_checkSpacefulnessCfg23= verifyNot checkSpacefulnessCfg "a=(1); echo ${a[@]}" -prop_checkSpacefulnessCfg24= verify checkSpacefulnessCfg "a='a b'; cat <<< $a" -prop_checkSpacefulnessCfg25= verify checkSpacefulnessCfg "a='s/[0-9]//g'; sed $a" -prop_checkSpacefulnessCfg26= verify checkSpacefulnessCfg "a='foo bar'; echo {1,2,$a}" -prop_checkSpacefulnessCfg27= verifyNot checkSpacefulnessCfg "echo ${a:+'foo'}" -prop_checkSpacefulnessCfg28= verifyNot checkSpacefulnessCfg "exec {n}>&1; echo $n" -prop_checkSpacefulnessCfg29= verifyNot checkSpacefulnessCfg "n=$(stuff); exec {n}>&-;" -prop_checkSpacefulnessCfg30= verify checkSpacefulnessCfg "file='foo bar'; echo foo > $file;" -prop_checkSpacefulnessCfg31= verifyNot checkSpacefulnessCfg "echo \"`echo \\\"$1\\\"`\"" -prop_checkSpacefulnessCfg32= verifyNot checkSpacefulnessCfg "var=$1; [ -v var ]" -prop_checkSpacefulnessCfg33= verify checkSpacefulnessCfg "for file; do echo $file; done" -prop_checkSpacefulnessCfg34= verify checkSpacefulnessCfg "declare foo$n=$1" -prop_checkSpacefulnessCfg35= verifyNot checkSpacefulnessCfg "echo ${1+\"$1\"}" -prop_checkSpacefulnessCfg36= verifyNot checkSpacefulnessCfg "arg=$#; echo $arg" -prop_checkSpacefulnessCfg37= verifyNot checkSpacefulnessCfg "@test 'status' {\n [ $status -eq 0 ]\n}" +prop_checkSpacefulnessCfg10 = verify checkSpacefulnessCfg "rm $1" +prop_checkSpacefulnessCfg11 = verify checkSpacefulnessCfg "rm ${10//foo/bar}" +prop_checkSpacefulnessCfg12 = verifyNot checkSpacefulnessCfg "(( $1 + 3 ))" +prop_checkSpacefulnessCfg13 = verifyNot checkSpacefulnessCfg "if [[ $2 -gt 14 ]]; then true; fi" +prop_checkSpacefulnessCfg14 = verifyNot checkSpacefulnessCfg "foo=$3 env" +prop_checkSpacefulnessCfg15 = verifyNot checkSpacefulnessCfg "local foo=$1" +prop_checkSpacefulnessCfg16 = verifyNot checkSpacefulnessCfg "declare foo=$1" +prop_checkSpacefulnessCfg17 = verify checkSpacefulnessCfg "echo foo=$1" +prop_checkSpacefulnessCfg18 = verifyNot checkSpacefulnessCfg "$1 --flags" +prop_checkSpacefulnessCfg19 = verify checkSpacefulnessCfg "echo $PWD" +prop_checkSpacefulnessCfg20 = verifyNot checkSpacefulnessCfg "n+='foo bar'" +prop_checkSpacefulnessCfg21 = verifyNot checkSpacefulnessCfg "select foo in $bar; do true; done" +prop_checkSpacefulnessCfg22 = verifyNot checkSpacefulnessCfg "echo $\"$1\"" +prop_checkSpacefulnessCfg23 = verifyNot checkSpacefulnessCfg "a=(1); echo ${a[@]}" +prop_checkSpacefulnessCfg24 = verify checkSpacefulnessCfg "a='a b'; cat <<< $a" +prop_checkSpacefulnessCfg25 = verify checkSpacefulnessCfg "a='s/[0-9]//g'; sed $a" +prop_checkSpacefulnessCfg26 = verify checkSpacefulnessCfg "a='foo bar'; echo {1,2,$a}" +prop_checkSpacefulnessCfg27 = verifyNot checkSpacefulnessCfg "echo ${a:+'foo'}" +prop_checkSpacefulnessCfg28 = verifyNot checkSpacefulnessCfg "exec {n}>&1; echo $n" +prop_checkSpacefulnessCfg29 = verifyNot checkSpacefulnessCfg "n=$(stuff); exec {n}>&-;" +prop_checkSpacefulnessCfg30 = verify checkSpacefulnessCfg "file='foo bar'; echo foo > $file;" +prop_checkSpacefulnessCfg31 = verifyNot checkSpacefulnessCfg "echo \"`echo \\\"$1\\\"`\"" +prop_checkSpacefulnessCfg32 = verifyNot checkSpacefulnessCfg "var=$1; [ -v var ]" +prop_checkSpacefulnessCfg33 = verify checkSpacefulnessCfg "for file; do echo $file; done" +prop_checkSpacefulnessCfg34 = verify checkSpacefulnessCfg "declare foo$n=$1" +prop_checkSpacefulnessCfg35 = verifyNot checkSpacefulnessCfg "echo ${1+\"$1\"}" +prop_checkSpacefulnessCfg36 = verifyNot checkSpacefulnessCfg "arg=$#; echo $arg" +prop_checkSpacefulnessCfg37 = verifyNot checkSpacefulnessCfg "@test 'status' {\n [ $status -eq 0 ]\n}" prop_checkSpacefulnessCfg37v = verify checkVerboseSpacefulnessCfg "@test 'status' {\n [ $status -eq 0 ]\n}" -prop_checkSpacefulnessCfg38= verify checkSpacefulnessCfg "a=; echo $a" -prop_checkSpacefulnessCfg39= verifyNot checkSpacefulnessCfg "a=''\"\"''; b=x$a; echo $b" -prop_checkSpacefulnessCfg40= verifyNot checkSpacefulnessCfg "a=$((x+1)); echo $a" -prop_checkSpacefulnessCfg41= verifyNot checkSpacefulnessCfg "exec $1 --flags" -prop_checkSpacefulnessCfg42= verifyNot checkSpacefulnessCfg "run $1 --flags" -prop_checkSpacefulnessCfg43= verifyNot checkSpacefulnessCfg "$foo=42" -prop_checkSpacefulnessCfg44= verify checkSpacefulnessCfg "#!/bin/sh\nexport var=$value" -prop_checkSpacefulnessCfg45= verifyNot checkSpacefulnessCfg "wait -zzx -p foo; echo $foo" -prop_checkSpacefulnessCfg46= verifyNot checkSpacefulnessCfg "x=0; (( x += 1 )); echo $x" -prop_checkSpacefulnessCfg47= verifyNot checkSpacefulnessCfg "x=0; (( x-- )); echo $x" -prop_checkSpacefulnessCfg48= verifyNot checkSpacefulnessCfg "x=0; (( ++x )); echo $x" -prop_checkSpacefulnessCfg49= verifyNot checkSpacefulnessCfg "for i in 1 2 3; do echo $i; done" -prop_checkSpacefulnessCfg50= verify checkSpacefulnessCfg "for i in 1 2 *; do echo $i; done" -prop_checkSpacefulnessCfg51= verify checkSpacefulnessCfg "x='foo bar'; x && x=1; echo $x" -prop_checkSpacefulnessCfg52= verifyNot checkSpacefulnessCfg "x=1; if f; then x='foo bar'; exit; fi; echo $x" -prop_checkSpacefulnessCfg53= verifyNot checkSpacefulnessCfg "s=1; f() { local s='a b'; }; f; echo $s" -prop_checkSpacefulnessCfg54= verifyNot checkSpacefulnessCfg "s='a b'; f() { s=1; }; f; echo $s" -prop_checkSpacefulnessCfg55= verify checkSpacefulnessCfg "s='a b'; x && f() { s=1; }; f; echo $s" -prop_checkSpacefulnessCfg56= verifyNot checkSpacefulnessCfg "s=1; cat <(s='a b'); echo $s" -prop_checkSpacefulnessCfg57= verifyNot checkSpacefulnessCfg "declare -i s=0; s=$(f); echo $s" -prop_checkSpacefulnessCfg58= verify checkSpacefulnessCfg "f() { declare -i s; }; f; s=$(var); echo $s" -prop_checkSpacefulnessCfg59= verifyNot checkSpacefulnessCfg "f() { declare -gi s; }; f; s=$(var); echo $s" -prop_checkSpacefulnessCfg60= verify checkSpacefulnessCfg "declare -i s; declare +i s; s=$(foo); echo $s" -prop_checkSpacefulnessCfg61= verify checkSpacefulnessCfg "declare -x X; y=foo$X; echo $y;" -prop_checkSpacefulnessCfg62= verifyNot checkSpacefulnessCfg "f() { declare -x X; y=foo$X; echo $y; }" -prop_checkSpacefulnessCfg63= verify checkSpacefulnessCfg "f && declare -i s; s='x + y'; echo $s" -prop_checkSpacefulnessCfg64= verifyNot checkSpacefulnessCfg "declare -i s; s='x + y'; x=$s; echo $x" +prop_checkSpacefulnessCfg38 = verify checkSpacefulnessCfg "a=; echo $a" +prop_checkSpacefulnessCfg39 = verifyNot checkSpacefulnessCfg "a=''\"\"''; b=x$a; echo $b" +prop_checkSpacefulnessCfg40 = verifyNot checkSpacefulnessCfg "a=$((x+1)); echo $a" +prop_checkSpacefulnessCfg41 = verifyNot checkSpacefulnessCfg "exec $1 --flags" +prop_checkSpacefulnessCfg42 = verifyNot checkSpacefulnessCfg "run $1 --flags" +prop_checkSpacefulnessCfg43 = verifyNot checkSpacefulnessCfg "$foo=42" +prop_checkSpacefulnessCfg44 = verify checkSpacefulnessCfg "#!/bin/sh\nexport var=$value" +prop_checkSpacefulnessCfg45 = verifyNot checkSpacefulnessCfg "wait -zzx -p foo; echo $foo" +prop_checkSpacefulnessCfg46 = verifyNot checkSpacefulnessCfg "x=0; (( x += 1 )); echo $x" +prop_checkSpacefulnessCfg47 = verifyNot checkSpacefulnessCfg "x=0; (( x-- )); echo $x" +prop_checkSpacefulnessCfg48 = verifyNot checkSpacefulnessCfg "x=0; (( ++x )); echo $x" +prop_checkSpacefulnessCfg49 = verifyNot checkSpacefulnessCfg "for i in 1 2 3; do echo $i; done" +prop_checkSpacefulnessCfg50 = verify checkSpacefulnessCfg "for i in 1 2 *; do echo $i; done" +prop_checkSpacefulnessCfg51 = verify checkSpacefulnessCfg "x='foo bar'; x && x=1; echo $x" +prop_checkSpacefulnessCfg52 = verifyNot checkSpacefulnessCfg "x=1; if f; then x='foo bar'; exit; fi; echo $x" +prop_checkSpacefulnessCfg53 = verifyNot checkSpacefulnessCfg "s=1; f() { local s='a b'; }; f; echo $s" +prop_checkSpacefulnessCfg54 = verifyNot checkSpacefulnessCfg "s='a b'; f() { s=1; }; f; echo $s" +prop_checkSpacefulnessCfg55 = verify checkSpacefulnessCfg "s='a b'; x && f() { s=1; }; f; echo $s" +prop_checkSpacefulnessCfg56 = verifyNot checkSpacefulnessCfg "s=1; cat <(s='a b'); echo $s" +prop_checkSpacefulnessCfg57 = verifyNot checkSpacefulnessCfg "declare -i s=0; s=$(f); echo $s" +prop_checkSpacefulnessCfg58 = verify checkSpacefulnessCfg "f() { declare -i s; }; f; s=$(var); echo $s" +prop_checkSpacefulnessCfg59 = verifyNot checkSpacefulnessCfg "f() { declare -gi s; }; f; s=$(var); echo $s" +prop_checkSpacefulnessCfg60 = verify checkSpacefulnessCfg "declare -i s; declare +i s; s=$(foo); echo $s" +prop_checkSpacefulnessCfg61 = verify checkSpacefulnessCfg "declare -x X; y=foo$X; echo $y;" +prop_checkSpacefulnessCfg62 = verifyNot checkSpacefulnessCfg "f() { declare -x X; y=foo$X; echo $y; }" +prop_checkSpacefulnessCfg63 = verify checkSpacefulnessCfg "f && declare -i s; s='x + y'; echo $s" +prop_checkSpacefulnessCfg64 = verifyNot checkSpacefulnessCfg "declare -i s; s='x + y'; x=$s; echo $x" checkSpacefulnessCfg = checkSpacefulnessCfg' True checkVerboseSpacefulnessCfg = checkSpacefulnessCfg' False @@ -2157,13 +2157,13 @@ checkVariableBraces params t@(T_DollarBraced id False l) checkVariableBraces _ _ = return () prop_checkQuotesInLiterals1 = verifyTree checkQuotesInLiterals "param='--foo=\"bar\"'; app $param" -prop_checkQuotesInLiterals1a= verifyTree checkQuotesInLiterals "param=\"--foo='lolbar'\"; app $param" +prop_checkQuotesInLiterals1a = verifyTree checkQuotesInLiterals "param=\"--foo='lolbar'\"; app $param" prop_checkQuotesInLiterals2 = verifyNotTree checkQuotesInLiterals "param='--foo=\"bar\"'; app \"$param\"" prop_checkQuotesInLiterals3 =verifyNotTree checkQuotesInLiterals "param=('--foo='); app \"${param[@]}\"" prop_checkQuotesInLiterals4 = verifyNotTree checkQuotesInLiterals "param=\"don't bother with this one\"; app $param" prop_checkQuotesInLiterals5 = verifyNotTree checkQuotesInLiterals "param=\"--foo='lolbar'\"; eval app $param" prop_checkQuotesInLiterals6 = verifyTree checkQuotesInLiterals "param='my\\ file'; cmd=\"rm $param\"; $cmd" -prop_checkQuotesInLiterals6a= verifyNotTree checkQuotesInLiterals "param='my\\ file'; cmd=\"rm ${#param}\"; $cmd" +prop_checkQuotesInLiterals6a = verifyNotTree checkQuotesInLiterals "param='my\\ file'; cmd=\"rm ${#param}\"; $cmd" prop_checkQuotesInLiterals7 = verifyTree checkQuotesInLiterals "param='my\\ file'; rm $param" prop_checkQuotesInLiterals8 = verifyTree checkQuotesInLiterals "param=\"/foo/'bar baz'/etc\"; rm $param" prop_checkQuotesInLiterals9 = verifyNotTree checkQuotesInLiterals "param=\"/foo/'bar baz'/etc\"; rm ${#param}" @@ -2227,9 +2227,9 @@ prop_checkFunctionsUsedExternally1 = verifyTree checkFunctionsUsedExternally "foo() { :; }; sudo foo" prop_checkFunctionsUsedExternally2 = verifyTree checkFunctionsUsedExternally "alias f='a'; xargs -0 f" -prop_checkFunctionsUsedExternally2b= +prop_checkFunctionsUsedExternally2b = verifyNotTree checkFunctionsUsedExternally "alias f='a'; find . -type f" -prop_checkFunctionsUsedExternally2c= +prop_checkFunctionsUsedExternally2c = verifyTree checkFunctionsUsedExternally "alias f='a'; find . -type f -exec f +" prop_checkFunctionsUsedExternally3 = verifyNotTree checkFunctionsUsedExternally "f() { :; }; echo f" @@ -2305,49 +2305,49 @@ prop_checkUnused6 = verifyNotTree checkUnusedAssignments "var=4; (( var++ ))" prop_checkUnused7 = verifyNotTree checkUnusedAssignments "var=2; $((var))" prop_checkUnused8 = verifyTree checkUnusedAssignments "var=2; var=3;" prop_checkUnused9 = verifyNotTree checkUnusedAssignments "read ''" -prop_checkUnused10= verifyNotTree checkUnusedAssignments "read -p 'test: '" -prop_checkUnused11= verifyNotTree checkUnusedAssignments "bar=5; export foo[$bar]=3" -prop_checkUnused12= verifyNotTree checkUnusedAssignments "read foo; echo ${!foo}" -prop_checkUnused13= verifyNotTree checkUnusedAssignments "x=(1); (( x[0] ))" -prop_checkUnused14= verifyNotTree checkUnusedAssignments "x=(1); n=0; echo ${x[n]}" -prop_checkUnused15= verifyNotTree checkUnusedAssignments "x=(1); n=0; (( x[n] ))" -prop_checkUnused16= verifyNotTree checkUnusedAssignments "foo=5; declare -x foo" -prop_checkUnused16b= verifyNotTree checkUnusedAssignments "f() { local -x foo; foo=42; bar; }; f" -prop_checkUnused17= verifyNotTree checkUnusedAssignments "read -i 'foo' -e -p 'Input: ' bar; $bar;" -prop_checkUnused18= verifyNotTree checkUnusedAssignments "a=1; arr=( [$a]=42 ); echo \"${arr[@]}\"" -prop_checkUnused19= verifyNotTree checkUnusedAssignments "a=1; let b=a+1; echo $b" -prop_checkUnused20= verifyNotTree checkUnusedAssignments "a=1; PS1='$a'" -prop_checkUnused21= verifyNotTree checkUnusedAssignments "a=1; trap 'echo $a' INT" -prop_checkUnused22= verifyNotTree checkUnusedAssignments "a=1; [ -v a ]" -prop_checkUnused23= verifyNotTree checkUnusedAssignments "a=1; [ -R a ]" -prop_checkUnused24= verifyNotTree checkUnusedAssignments "mapfile -C a b; echo ${b[@]}" -prop_checkUnused25= verifyNotTree checkUnusedAssignments "readarray foo; echo ${foo[@]}" -prop_checkUnused26= verifyNotTree checkUnusedAssignments "declare -F foo" -prop_checkUnused27= verifyTree checkUnusedAssignments "var=3; [ var -eq 3 ]" -prop_checkUnused28= verifyNotTree checkUnusedAssignments "var=3; [[ var -eq 3 ]]" -prop_checkUnused29= verifyNotTree checkUnusedAssignments "var=(a b); declare -p var" -prop_checkUnused30= verifyTree checkUnusedAssignments "let a=1" -prop_checkUnused31= verifyTree checkUnusedAssignments "let 'a=1'" -prop_checkUnused32= verifyTree checkUnusedAssignments "let a=b=c; echo $a" -prop_checkUnused33= verifyNotTree checkUnusedAssignments "a=foo; [[ foo =~ ^{$a}$ ]]" -prop_checkUnused34= verifyNotTree checkUnusedAssignments "foo=1; (( t = foo )); echo $t" -prop_checkUnused35= verifyNotTree checkUnusedAssignments "a=foo; b=2; echo ${a:b}" -prop_checkUnused36= verifyNotTree checkUnusedAssignments "if [[ -v foo ]]; then true; fi" -prop_checkUnused37= verifyNotTree checkUnusedAssignments "fd=2; exec {fd}>&-" -prop_checkUnused38= verifyTree checkUnusedAssignments "(( a=42 ))" -prop_checkUnused39= verifyNotTree checkUnusedAssignments "declare -x -f foo" -prop_checkUnused40= verifyNotTree checkUnusedAssignments "arr=(1 2); num=2; echo \"${arr[@]:num}\"" -prop_checkUnused41= verifyNotTree checkUnusedAssignments "@test 'foo' {\ntrue\n}\n" -prop_checkUnused42= verifyNotTree checkUnusedAssignments "DEFINE_string foo '' ''; echo \"${FLAGS_foo}\"" -prop_checkUnused43= verifyTree checkUnusedAssignments "DEFINE_string foo '' ''" -prop_checkUnused44= verifyNotTree checkUnusedAssignments "DEFINE_string \"foo$ibar\" x y" -prop_checkUnused45= verifyTree checkUnusedAssignments "readonly foo=bar" -prop_checkUnused46= verifyTree checkUnusedAssignments "readonly foo=(bar)" -prop_checkUnused47= verifyNotTree checkUnusedAssignments "a=1; alias hello='echo $a'" -prop_checkUnused48= verifyNotTree checkUnusedAssignments "_a=1" -prop_checkUnused49= verifyNotTree checkUnusedAssignments "declare -A array; key=a; [[ -v array[$key] ]]" -prop_checkUnused50= verifyNotTree checkUnusedAssignments "foofunc() { :; }; typeset -fx foofunc" -prop_checkUnused51= verifyTree checkUnusedAssignments "x[y[z=1]]=1; echo ${x[@]}" +prop_checkUnused10 = verifyNotTree checkUnusedAssignments "read -p 'test: '" +prop_checkUnused11 = verifyNotTree checkUnusedAssignments "bar=5; export foo[$bar]=3" +prop_checkUnused12 = verifyNotTree checkUnusedAssignments "read foo; echo ${!foo}" +prop_checkUnused13 = verifyNotTree checkUnusedAssignments "x=(1); (( x[0] ))" +prop_checkUnused14 = verifyNotTree checkUnusedAssignments "x=(1); n=0; echo ${x[n]}" +prop_checkUnused15 = verifyNotTree checkUnusedAssignments "x=(1); n=0; (( x[n] ))" +prop_checkUnused16 = verifyNotTree checkUnusedAssignments "foo=5; declare -x foo" +prop_checkUnused16b = verifyNotTree checkUnusedAssignments "f() { local -x foo; foo=42; bar; }; f" +prop_checkUnused17 = verifyNotTree checkUnusedAssignments "read -i 'foo' -e -p 'Input: ' bar; $bar;" +prop_checkUnused18 = verifyNotTree checkUnusedAssignments "a=1; arr=( [$a]=42 ); echo \"${arr[@]}\"" +prop_checkUnused19 = verifyNotTree checkUnusedAssignments "a=1; let b=a+1; echo $b" +prop_checkUnused20 = verifyNotTree checkUnusedAssignments "a=1; PS1='$a'" +prop_checkUnused21 = verifyNotTree checkUnusedAssignments "a=1; trap 'echo $a' INT" +prop_checkUnused22 = verifyNotTree checkUnusedAssignments "a=1; [ -v a ]" +prop_checkUnused23 = verifyNotTree checkUnusedAssignments "a=1; [ -R a ]" +prop_checkUnused24 = verifyNotTree checkUnusedAssignments "mapfile -C a b; echo ${b[@]}" +prop_checkUnused25 = verifyNotTree checkUnusedAssignments "readarray foo; echo ${foo[@]}" +prop_checkUnused26 = verifyNotTree checkUnusedAssignments "declare -F foo" +prop_checkUnused27 = verifyTree checkUnusedAssignments "var=3; [ var -eq 3 ]" +prop_checkUnused28 = verifyNotTree checkUnusedAssignments "var=3; [[ var -eq 3 ]]" +prop_checkUnused29 = verifyNotTree checkUnusedAssignments "var=(a b); declare -p var" +prop_checkUnused30 = verifyTree checkUnusedAssignments "let a=1" +prop_checkUnused31 = verifyTree checkUnusedAssignments "let 'a=1'" +prop_checkUnused32 = verifyTree checkUnusedAssignments "let a=b=c; echo $a" +prop_checkUnused33 = verifyNotTree checkUnusedAssignments "a=foo; [[ foo =~ ^{$a}$ ]]" +prop_checkUnused34 = verifyNotTree checkUnusedAssignments "foo=1; (( t = foo )); echo $t" +prop_checkUnused35 = verifyNotTree checkUnusedAssignments "a=foo; b=2; echo ${a:b}" +prop_checkUnused36 = verifyNotTree checkUnusedAssignments "if [[ -v foo ]]; then true; fi" +prop_checkUnused37 = verifyNotTree checkUnusedAssignments "fd=2; exec {fd}>&-" +prop_checkUnused38 = verifyTree checkUnusedAssignments "(( a=42 ))" +prop_checkUnused39 = verifyNotTree checkUnusedAssignments "declare -x -f foo" +prop_checkUnused40 = verifyNotTree checkUnusedAssignments "arr=(1 2); num=2; echo \"${arr[@]:num}\"" +prop_checkUnused41 = verifyNotTree checkUnusedAssignments "@test 'foo' {\ntrue\n}\n" +prop_checkUnused42 = verifyNotTree checkUnusedAssignments "DEFINE_string foo '' ''; echo \"${FLAGS_foo}\"" +prop_checkUnused43 = verifyTree checkUnusedAssignments "DEFINE_string foo '' ''" +prop_checkUnused44 = verifyNotTree checkUnusedAssignments "DEFINE_string \"foo$ibar\" x y" +prop_checkUnused45 = verifyTree checkUnusedAssignments "readonly foo=bar" +prop_checkUnused46 = verifyTree checkUnusedAssignments "readonly foo=(bar)" +prop_checkUnused47 = verifyNotTree checkUnusedAssignments "a=1; alias hello='echo $a'" +prop_checkUnused48 = verifyNotTree checkUnusedAssignments "_a=1" +prop_checkUnused49 = verifyNotTree checkUnusedAssignments "declare -A array; key=a; [[ -v array[$key] ]]" +prop_checkUnused50 = verifyNotTree checkUnusedAssignments "foofunc() { :; }; typeset -fx foofunc" +prop_checkUnused51 = verifyTree checkUnusedAssignments "x[y[z=1]]=1; echo ${x[@]}" checkUnusedAssignments params t = execWriter (mapM_ warnFor unused) where @@ -2381,40 +2381,40 @@ prop_checkUnassignedReferences6 = verifyNotTree checkUnassignedReferences "foo=. prop_checkUnassignedReferences7 = verifyNotTree checkUnassignedReferences "getopts ':h' foo; echo $foo" prop_checkUnassignedReferences8 = verifyNotTree checkUnassignedReferences "let 'foo = 1'; echo $foo" prop_checkUnassignedReferences9 = verifyNotTree checkUnassignedReferences "echo ${foo-bar}" -prop_checkUnassignedReferences10= verifyNotTree checkUnassignedReferences "echo ${foo:?}" -prop_checkUnassignedReferences11= verifyNotTree checkUnassignedReferences "declare -A foo; echo \"${foo[@]}\"" -prop_checkUnassignedReferences12= verifyNotTree checkUnassignedReferences "typeset -a foo; echo \"${foo[@]}\"" -prop_checkUnassignedReferences13= verifyNotTree checkUnassignedReferences "f() { local foo; echo $foo; }" -prop_checkUnassignedReferences14= verifyNotTree checkUnassignedReferences "foo=; echo $foo" -prop_checkUnassignedReferences15= verifyNotTree checkUnassignedReferences "f() { true; }; export -f f" -prop_checkUnassignedReferences16= verifyNotTree checkUnassignedReferences "declare -A foo=( [a b]=bar ); echo ${foo[a b]}" -prop_checkUnassignedReferences17= verifyNotTree checkUnassignedReferences "USERS=foo; echo $USER" -prop_checkUnassignedReferences18= verifyNotTree checkUnassignedReferences "FOOBAR=42; export FOOBAR=" -prop_checkUnassignedReferences19= verifyNotTree checkUnassignedReferences "readonly foo=bar; echo $foo" -prop_checkUnassignedReferences20= verifyNotTree checkUnassignedReferences "printf -v foo bar; echo $foo" -prop_checkUnassignedReferences21= verifyTree checkUnassignedReferences "echo ${#foo}" -prop_checkUnassignedReferences22= verifyNotTree checkUnassignedReferences "echo ${!os*}" -prop_checkUnassignedReferences23= verifyTree checkUnassignedReferences "declare -a foo; foo[bar]=42;" -prop_checkUnassignedReferences24= verifyNotTree checkUnassignedReferences "declare -A foo; foo[bar]=42;" -prop_checkUnassignedReferences25= verifyNotTree checkUnassignedReferences "declare -A foo=(); foo[bar]=42;" -prop_checkUnassignedReferences26= verifyNotTree checkUnassignedReferences "a::b() { foo; }; readonly -f a::b" -prop_checkUnassignedReferences27= verifyNotTree checkUnassignedReferences ": ${foo:=bar}" -prop_checkUnassignedReferences28= verifyNotTree checkUnassignedReferences "#!/bin/ksh\necho \"${.sh.version}\"\n" -prop_checkUnassignedReferences29= verifyNotTree checkUnassignedReferences "if [[ -v foo ]]; then echo $foo; fi" -prop_checkUnassignedReferences30= verifyNotTree checkUnassignedReferences "if [[ -v foo[3] ]]; then echo ${foo[3]}; fi" -prop_checkUnassignedReferences31= verifyNotTree checkUnassignedReferences "X=1; if [[ -v foo[$X+42] ]]; then echo ${foo[$X+42]}; fi" -prop_checkUnassignedReferences32= verifyNotTree checkUnassignedReferences "if [[ -v \"foo[1]\" ]]; then echo ${foo[@]}; fi" -prop_checkUnassignedReferences33= verifyNotTree checkUnassignedReferences "f() { local -A foo; echo \"${foo[@]}\"; }" -prop_checkUnassignedReferences34= verifyNotTree checkUnassignedReferences "declare -A foo; (( foo[bar] ))" -prop_checkUnassignedReferences35= verifyNotTree checkUnassignedReferences "echo ${arr[foo-bar]:?fail}" -prop_checkUnassignedReferences36= verifyNotTree checkUnassignedReferences "read -a foo -r <<<\"foo bar\"; echo \"$foo\"" -prop_checkUnassignedReferences37= verifyNotTree checkUnassignedReferences "var=howdy; printf -v 'array[0]' %s \"$var\"; printf %s \"${array[0]}\";" -prop_checkUnassignedReferences38= verifyTree (checkUnassignedReferences' True) "echo $VAR" -prop_checkUnassignedReferences39= verifyNotTree checkUnassignedReferences "builtin export var=4; echo $var" -prop_checkUnassignedReferences40= verifyNotTree checkUnassignedReferences ": ${foo=bar}" -prop_checkUnassignedReferences41= verifyNotTree checkUnassignedReferences "mapfile -t files 123; echo \"${files[@]}\"" -prop_checkUnassignedReferences42= verifyNotTree checkUnassignedReferences "mapfile files -t; echo \"${files[@]}\"" -prop_checkUnassignedReferences43= verifyNotTree checkUnassignedReferences "mapfile --future files; echo \"${files[@]}\"" +prop_checkUnassignedReferences10 = verifyNotTree checkUnassignedReferences "echo ${foo:?}" +prop_checkUnassignedReferences11 = verifyNotTree checkUnassignedReferences "declare -A foo; echo \"${foo[@]}\"" +prop_checkUnassignedReferences12 = verifyNotTree checkUnassignedReferences "typeset -a foo; echo \"${foo[@]}\"" +prop_checkUnassignedReferences13 = verifyNotTree checkUnassignedReferences "f() { local foo; echo $foo; }" +prop_checkUnassignedReferences14 = verifyNotTree checkUnassignedReferences "foo=; echo $foo" +prop_checkUnassignedReferences15 = verifyNotTree checkUnassignedReferences "f() { true; }; export -f f" +prop_checkUnassignedReferences16 = verifyNotTree checkUnassignedReferences "declare -A foo=( [a b]=bar ); echo ${foo[a b]}" +prop_checkUnassignedReferences17 = verifyNotTree checkUnassignedReferences "USERS=foo; echo $USER" +prop_checkUnassignedReferences18 = verifyNotTree checkUnassignedReferences "FOOBAR=42; export FOOBAR=" +prop_checkUnassignedReferences19 = verifyNotTree checkUnassignedReferences "readonly foo=bar; echo $foo" +prop_checkUnassignedReferences20 = verifyNotTree checkUnassignedReferences "printf -v foo bar; echo $foo" +prop_checkUnassignedReferences21 = verifyTree checkUnassignedReferences "echo ${#foo}" +prop_checkUnassignedReferences22 = verifyNotTree checkUnassignedReferences "echo ${!os*}" +prop_checkUnassignedReferences23 = verifyTree checkUnassignedReferences "declare -a foo; foo[bar]=42;" +prop_checkUnassignedReferences24 = verifyNotTree checkUnassignedReferences "declare -A foo; foo[bar]=42;" +prop_checkUnassignedReferences25 = verifyNotTree checkUnassignedReferences "declare -A foo=(); foo[bar]=42;" +prop_checkUnassignedReferences26 = verifyNotTree checkUnassignedReferences "a::b() { foo; }; readonly -f a::b" +prop_checkUnassignedReferences27 = verifyNotTree checkUnassignedReferences ": ${foo:=bar}" +prop_checkUnassignedReferences28 = verifyNotTree checkUnassignedReferences "#!/bin/ksh\necho \"${.sh.version}\"\n" +prop_checkUnassignedReferences29 = verifyNotTree checkUnassignedReferences "if [[ -v foo ]]; then echo $foo; fi" +prop_checkUnassignedReferences30 = verifyNotTree checkUnassignedReferences "if [[ -v foo[3] ]]; then echo ${foo[3]}; fi" +prop_checkUnassignedReferences31 = verifyNotTree checkUnassignedReferences "X=1; if [[ -v foo[$X+42] ]]; then echo ${foo[$X+42]}; fi" +prop_checkUnassignedReferences32 = verifyNotTree checkUnassignedReferences "if [[ -v \"foo[1]\" ]]; then echo ${foo[@]}; fi" +prop_checkUnassignedReferences33 = verifyNotTree checkUnassignedReferences "f() { local -A foo; echo \"${foo[@]}\"; }" +prop_checkUnassignedReferences34 = verifyNotTree checkUnassignedReferences "declare -A foo; (( foo[bar] ))" +prop_checkUnassignedReferences35 = verifyNotTree checkUnassignedReferences "echo ${arr[foo-bar]:?fail}" +prop_checkUnassignedReferences36 = verifyNotTree checkUnassignedReferences "read -a foo -r <<<\"foo bar\"; echo \"$foo\"" +prop_checkUnassignedReferences37 = verifyNotTree checkUnassignedReferences "var=howdy; printf -v 'array[0]' %s \"$var\"; printf %s \"${array[0]}\";" +prop_checkUnassignedReferences38 = verifyTree (checkUnassignedReferences' True) "echo $VAR" +prop_checkUnassignedReferences39 = verifyNotTree checkUnassignedReferences "builtin export var=4; echo $var" +prop_checkUnassignedReferences40 = verifyNotTree checkUnassignedReferences ": ${foo=bar}" +prop_checkUnassignedReferences41 = verifyNotTree checkUnassignedReferences "mapfile -t files 123; echo \"${files[@]}\"" +prop_checkUnassignedReferences42 = verifyNotTree checkUnassignedReferences "mapfile files -t; echo \"${files[@]}\"" +prop_checkUnassignedReferences43 = verifyNotTree checkUnassignedReferences "mapfile --future files; echo \"${files[@]}\"" prop_checkUnassignedReferences_minusNPlain = verifyNotTree checkUnassignedReferences "if [ -n \"$x\" ]; then echo $x; fi" prop_checkUnassignedReferences_minusZPlain = verifyNotTree checkUnassignedReferences "if [ -z \"$x\" ]; then echo \"\"; fi" prop_checkUnassignedReferences_minusNBraced = verifyNotTree checkUnassignedReferences "if [ -n \"${x}\" ]; then echo $x; fi" @@ -2792,11 +2792,11 @@ prop_checkUnpassedInFunctions6 = verifyNotTree checkUnpassedInFunctions "foo() { prop_checkUnpassedInFunctions7 = verifyTree checkUnpassedInFunctions "foo() { echo $1; }; foo; foo;" prop_checkUnpassedInFunctions8 = verifyNotTree checkUnpassedInFunctions "foo() { echo $((1)); }; foo;" prop_checkUnpassedInFunctions9 = verifyNotTree checkUnpassedInFunctions "foo() { echo $(($b)); }; foo;" -prop_checkUnpassedInFunctions10= verifyNotTree checkUnpassedInFunctions "foo() { echo $!; }; foo;" -prop_checkUnpassedInFunctions11= verifyNotTree checkUnpassedInFunctions "foo() { bar() { echo $1; }; bar baz; }; foo;" -prop_checkUnpassedInFunctions12= verifyNotTree checkUnpassedInFunctions "foo() { echo ${!var*}; }; foo;" -prop_checkUnpassedInFunctions13= verifyNotTree checkUnpassedInFunctions "# shellcheck disable=SC2120\nfoo() { echo $1; }\nfoo\n" -prop_checkUnpassedInFunctions14= verifyTree checkUnpassedInFunctions "foo() { echo $#; }; foo" +prop_checkUnpassedInFunctions10 = verifyNotTree checkUnpassedInFunctions "foo() { echo $!; }; foo;" +prop_checkUnpassedInFunctions11 = verifyNotTree checkUnpassedInFunctions "foo() { bar() { echo $1; }; bar baz; }; foo;" +prop_checkUnpassedInFunctions12 = verifyNotTree checkUnpassedInFunctions "foo() { echo ${!var*}; }; foo;" +prop_checkUnpassedInFunctions13 = verifyNotTree checkUnpassedInFunctions "# shellcheck disable=SC2120\nfoo() { echo $1; }\nfoo\n" +prop_checkUnpassedInFunctions14 = verifyTree checkUnpassedInFunctions "foo() { echo $#; }; foo" checkUnpassedInFunctions params root = execWriter $ mapM_ warnForGroup referenceGroups where @@ -2971,12 +2971,12 @@ checkSuspiciousIFS params (T_Assignment _ _ "IFS" [] value) = checkSuspiciousIFS _ _ = return () -prop_checkGrepQ1= verify checkShouldUseGrepQ "[[ $(foo | grep bar) ]]" -prop_checkGrepQ2= verify checkShouldUseGrepQ "[ -z $(fgrep lol) ]" -prop_checkGrepQ3= verify checkShouldUseGrepQ "[ -n \"$(foo | zgrep lol)\" ]" -prop_checkGrepQ4= verifyNot checkShouldUseGrepQ "[ -z $(grep bar | cmd) ]" -prop_checkGrepQ5= verifyNot checkShouldUseGrepQ "rm $(ls | grep file)" -prop_checkGrepQ6= verifyNot checkShouldUseGrepQ "[[ -n $(pgrep foo) ]]" +prop_checkGrepQ1 = verify checkShouldUseGrepQ "[[ $(foo | grep bar) ]]" +prop_checkGrepQ2 = verify checkShouldUseGrepQ "[ -z $(fgrep lol) ]" +prop_checkGrepQ3 = verify checkShouldUseGrepQ "[ -n \"$(foo | zgrep lol)\" ]" +prop_checkGrepQ4 = verifyNot checkShouldUseGrepQ "[ -z $(grep bar | cmd) ]" +prop_checkGrepQ5 = verifyNot checkShouldUseGrepQ "rm $(ls | grep file)" +prop_checkGrepQ6 = verifyNot checkShouldUseGrepQ "[[ -n $(pgrep foo) ]]" checkShouldUseGrepQ params t = sequence_ $ case t of TC_Nullary id _ token -> check id True token diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index acbf967..4bae17e 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -216,18 +216,18 @@ checker spec params = getChecker $ commandChecks ++ optionals prop_checkTr1 = verify checkTr "tr [a-f] [A-F]" prop_checkTr2 = verify checkTr "tr 'a-z' 'A-Z'" -prop_checkTr2a= verify checkTr "tr '[a-z]' '[A-Z]'" +prop_checkTr2a = verify checkTr "tr '[a-z]' '[A-Z]'" prop_checkTr3 = verifyNot checkTr "tr -d '[:lower:]'" -prop_checkTr3a= verifyNot checkTr "tr -d '[:upper:]'" -prop_checkTr3b= verifyNot checkTr "tr -d '|/_[:upper:]'" +prop_checkTr3a = verifyNot checkTr "tr -d '[:upper:]'" +prop_checkTr3b = verifyNot checkTr "tr -d '|/_[:upper:]'" prop_checkTr4 = verifyNot checkTr "ls [a-z]" prop_checkTr5 = verify checkTr "tr foo bar" prop_checkTr6 = verify checkTr "tr 'hello' 'world'" prop_checkTr8 = verifyNot checkTr "tr aeiou _____" prop_checkTr9 = verifyNot checkTr "a-z n-za-m" -prop_checkTr10= verifyNot checkTr "tr --squeeze-repeats rl lr" -prop_checkTr11= verifyNot checkTr "tr abc '[d*]'" -prop_checkTr12= verifyNot checkTr "tr '[=e=]' 'e'" +prop_checkTr10 = verifyNot checkTr "tr --squeeze-repeats rl lr" +prop_checkTr11 = verifyNot checkTr "tr abc '[d*]'" +prop_checkTr12 = verifyNot checkTr "tr '[=e=]' 'e'" checkTr = CommandCheck (Basename "tr") (mapM_ f . arguments) where f w | isGlob w = -- The user will go [ab] -> '[ab]' -> 'ab'. Fixme? @@ -339,20 +339,20 @@ prop_checkGrepRe6 = verifyNot checkGrepRe "grep foo \\*.mp3" prop_checkGrepRe7 = verify checkGrepRe "grep *foo* file" prop_checkGrepRe8 = verify checkGrepRe "ls | grep foo*.jpg" prop_checkGrepRe9 = verifyNot checkGrepRe "grep '[0-9]*' file" -prop_checkGrepRe10= verifyNot checkGrepRe "grep '^aa*' file" -prop_checkGrepRe11= verifyNot checkGrepRe "grep --include=*.png foo" -prop_checkGrepRe12= verifyNot checkGrepRe "grep -F 'Foo*' file" -prop_checkGrepRe13= verifyNot checkGrepRe "grep -- -foo bar*" -prop_checkGrepRe14= verifyNot checkGrepRe "grep -e -foo bar*" -prop_checkGrepRe15= verifyNot checkGrepRe "grep --regex -foo bar*" -prop_checkGrepRe16= verifyNot checkGrepRe "grep --include 'Foo*' file" -prop_checkGrepRe17= verifyNot checkGrepRe "grep --exclude 'Foo*' file" -prop_checkGrepRe18= verifyNot checkGrepRe "grep --exclude-dir 'Foo*' file" -prop_checkGrepRe19= verify checkGrepRe "grep -- 'Foo*' file" -prop_checkGrepRe20= verifyNot checkGrepRe "grep --fixed-strings 'Foo*' file" -prop_checkGrepRe21= verifyNot checkGrepRe "grep -o 'x*' file" -prop_checkGrepRe22= verifyNot checkGrepRe "grep --only-matching 'x*' file" -prop_checkGrepRe23= verifyNot checkGrepRe "grep '.*' file" +prop_checkGrepRe10 = verifyNot checkGrepRe "grep '^aa*' file" +prop_checkGrepRe11 = verifyNot checkGrepRe "grep --include=*.png foo" +prop_checkGrepRe12 = verifyNot checkGrepRe "grep -F 'Foo*' file" +prop_checkGrepRe13 = verifyNot checkGrepRe "grep -- -foo bar*" +prop_checkGrepRe14 = verifyNot checkGrepRe "grep -e -foo bar*" +prop_checkGrepRe15 = verifyNot checkGrepRe "grep --regex -foo bar*" +prop_checkGrepRe16 = verifyNot checkGrepRe "grep --include 'Foo*' file" +prop_checkGrepRe17 = verifyNot checkGrepRe "grep --exclude 'Foo*' file" +prop_checkGrepRe18 = verifyNot checkGrepRe "grep --exclude-dir 'Foo*' file" +prop_checkGrepRe19 = verify checkGrepRe "grep -- 'Foo*' file" +prop_checkGrepRe20 = verifyNot checkGrepRe "grep --fixed-strings 'Foo*' file" +prop_checkGrepRe21 = verifyNot checkGrepRe "grep -o 'x*' file" +prop_checkGrepRe22 = verifyNot checkGrepRe "grep --only-matching 'x*' file" +prop_checkGrepRe23 = verifyNot checkGrepRe "grep '.*' file" checkGrepRe = CommandCheck (Basename "grep") check where check cmd = f cmd (arguments cmd) @@ -400,7 +400,7 @@ checkGrepRe = CommandCheck (Basename "grep") check where prop_checkTrapQuotes1 = verify checkTrapQuotes "trap \"echo $num\" INT" -prop_checkTrapQuotes1a= verify checkTrapQuotes "trap \"echo `ls`\" INT" +prop_checkTrapQuotes1a = verify checkTrapQuotes "trap \"echo `ls`\" INT" prop_checkTrapQuotes2 = verifyNot checkTrapQuotes "trap 'echo $num' INT" prop_checkTrapQuotes3 = verify checkTrapQuotes "trap \"echo $((1+num))\" EXIT DEBUG" checkTrapQuotes = CommandCheck (Exactly "trap") (f . arguments) where @@ -657,19 +657,19 @@ prop_checkPrintfVar6 = verify checkPrintfVar "printf foo bar baz" prop_checkPrintfVar7 = verify checkPrintfVar "printf -- foo bar baz" prop_checkPrintfVar8 = verifyNot checkPrintfVar "printf '%s %s %s' \"${var[@]}\"" prop_checkPrintfVar9 = verifyNot checkPrintfVar "printf '%s %s %s\\n' *.png" -prop_checkPrintfVar10= verifyNot checkPrintfVar "printf '%s %s %s' foo bar baz" -prop_checkPrintfVar11= verifyNot checkPrintfVar "printf '%(%s%s)T' -1" -prop_checkPrintfVar12= verify checkPrintfVar "printf '%s %s\\n' 1 2 3" -prop_checkPrintfVar13= verifyNot checkPrintfVar "printf '%s %s\\n' 1 2 3 4" -prop_checkPrintfVar14= verify checkPrintfVar "printf '%*s\\n' 1" -prop_checkPrintfVar15= verifyNot checkPrintfVar "printf '%*s\\n' 1 2" -prop_checkPrintfVar16= verifyNot checkPrintfVar "printf $'string'" -prop_checkPrintfVar17= verify checkPrintfVar "printf '%-*s\\n' 1" -prop_checkPrintfVar18= verifyNot checkPrintfVar "printf '%-*s\\n' 1 2" -prop_checkPrintfVar19= verifyNot checkPrintfVar "printf '%(%s)T'" -prop_checkPrintfVar20= verifyNot checkPrintfVar "printf '%d %(%s)T' 42" -prop_checkPrintfVar21= verify checkPrintfVar "printf '%d %(%s)T'" -prop_checkPrintfVar22= verify checkPrintfVar "printf '%s\n%s' foo" +prop_checkPrintfVar10 = verifyNot checkPrintfVar "printf '%s %s %s' foo bar baz" +prop_checkPrintfVar11 = verifyNot checkPrintfVar "printf '%(%s%s)T' -1" +prop_checkPrintfVar12 = verify checkPrintfVar "printf '%s %s\\n' 1 2 3" +prop_checkPrintfVar13 = verifyNot checkPrintfVar "printf '%s %s\\n' 1 2 3 4" +prop_checkPrintfVar14 = verify checkPrintfVar "printf '%*s\\n' 1" +prop_checkPrintfVar15 = verifyNot checkPrintfVar "printf '%*s\\n' 1 2" +prop_checkPrintfVar16 = verifyNot checkPrintfVar "printf $'string'" +prop_checkPrintfVar17 = verify checkPrintfVar "printf '%-*s\\n' 1" +prop_checkPrintfVar18 = verifyNot checkPrintfVar "printf '%-*s\\n' 1 2" +prop_checkPrintfVar19 = verifyNot checkPrintfVar "printf '%(%s)T'" +prop_checkPrintfVar20 = verifyNot checkPrintfVar "printf '%d %(%s)T' 42" +prop_checkPrintfVar21 = verify checkPrintfVar "printf '%d %(%s)T'" +prop_checkPrintfVar22 = verify checkPrintfVar "printf '%s\n%s' foo" checkPrintfVar = CommandCheck (Exactly "printf") (f . arguments) where f (doubledash:rest) | getLiteralString doubledash == Just "--" = f rest @@ -1069,10 +1069,10 @@ prop_checkCatastrophicRm4 = verify checkCatastrophicRm "rm -fr /home/$(whoami)/* prop_checkCatastrophicRm5 = verifyNot checkCatastrophicRm "rm -r /home/${USER:-thing}/*" prop_checkCatastrophicRm6 = verify checkCatastrophicRm "rm --recursive /etc/*$config*" prop_checkCatastrophicRm8 = verify checkCatastrophicRm "rm -rf /home" -prop_checkCatastrophicRm10= verifyNot checkCatastrophicRm "rm -r \"${DIR}\"/{.gitignore,.gitattributes,ci}" -prop_checkCatastrophicRm11= verify checkCatastrophicRm "rm -r /{bin,sbin}/$exec" -prop_checkCatastrophicRm12= verify checkCatastrophicRm "rm -r /{{usr,},{bin,sbin}}/$exec" -prop_checkCatastrophicRm13= verifyNot checkCatastrophicRm "rm -r /{{a,b},{c,d}}/$exec" +prop_checkCatastrophicRm10 = verifyNot checkCatastrophicRm "rm -r \"${DIR}\"/{.gitignore,.gitattributes,ci}" +prop_checkCatastrophicRm11 = verify checkCatastrophicRm "rm -r /{bin,sbin}/$exec" +prop_checkCatastrophicRm12 = verify checkCatastrophicRm "rm -r /{{usr,},{bin,sbin}}/$exec" +prop_checkCatastrophicRm13 = verifyNot checkCatastrophicRm "rm -r /{{a,b},{c,d}}/$exec" prop_checkCatastrophicRmA = verify checkCatastrophicRm "rm -rf /usr /lib/nvidia-current/xorg/xorg" prop_checkCatastrophicRmB = verify checkCatastrophicRm "rm -rf \"$STEAMROOT/\"*" checkCatastrophicRm = CommandCheck (Basename "rm") $ \t -> diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 9ad17f5..c12da2d 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -92,55 +92,55 @@ prop_checkBashisms6 = verify checkBashisms "[ \"$a\" == 42 ]" prop_checkBashisms7 = verify checkBashisms "echo ${var[1]}" prop_checkBashisms8 = verify checkBashisms "echo ${!var[@]}" prop_checkBashisms9 = verify checkBashisms "echo ${!var*}" -prop_checkBashisms10= verify checkBashisms "echo ${var:4:12}" -prop_checkBashisms11= verifyNot checkBashisms "echo ${var:-4}" -prop_checkBashisms12= verify checkBashisms "echo ${var//foo/bar}" -prop_checkBashisms13= verify checkBashisms "exec -c env" -prop_checkBashisms14= verify checkBashisms "echo -n \"Foo: \"" -prop_checkBashisms15= verify checkBashisms "let n++" -prop_checkBashisms16= verify checkBashisms "echo $RANDOM" -prop_checkBashisms17= verify checkBashisms "echo $((RANDOM%6+1))" -prop_checkBashisms18= verify checkBashisms "foo &> /dev/null" -prop_checkBashisms19= verify checkBashisms "foo > file*.txt" -prop_checkBashisms20= verify checkBashisms "read -ra foo" -prop_checkBashisms21= verify checkBashisms "[ -a foo ]" -prop_checkBashisms22= verifyNot checkBashisms "[ foo -a bar ]" -prop_checkBashisms23= verify checkBashisms "trap mything ERR INT" -prop_checkBashisms24= verifyNot checkBashisms "trap mything INT TERM" -prop_checkBashisms25= verify checkBashisms "cat < /dev/tcp/host/123" -prop_checkBashisms26= verify checkBashisms "trap mything ERR SIGTERM" -prop_checkBashisms27= verify checkBashisms "echo *[^0-9]*" -prop_checkBashisms28= verify checkBashisms "exec {n}>&2" -prop_checkBashisms29= verify checkBashisms "echo ${!var}" -prop_checkBashisms30= verify checkBashisms "printf -v '%s' \"$1\"" -prop_checkBashisms31= verify checkBashisms "printf '%q' \"$1\"" -prop_checkBashisms32= verifyNot checkBashisms "#!/bin/dash\n[ foo -nt bar ]" -prop_checkBashisms33= verify checkBashisms "#!/bin/sh\necho -n foo" -prop_checkBashisms34= verifyNot checkBashisms "#!/bin/dash\necho -n foo" -prop_checkBashisms35= verifyNot checkBashisms "#!/bin/dash\nlocal foo" -prop_checkBashisms36= verifyNot checkBashisms "#!/bin/dash\nread -p foo -r bar" -prop_checkBashisms37= verifyNot checkBashisms "HOSTNAME=foo; echo $HOSTNAME" -prop_checkBashisms38= verify checkBashisms "RANDOM=9; echo $RANDOM" -prop_checkBashisms39= verify checkBashisms "foo-bar() { true; }" -prop_checkBashisms40= verify checkBashisms "echo $(/dev/null" -prop_checkBashisms48= verifyNot checkBashisms "#!/bin/sh\necho $LINENO" -prop_checkBashisms49= verify checkBashisms "#!/bin/dash\necho $MACHTYPE" -prop_checkBashisms50= verify checkBashisms "#!/bin/sh\ncmd >& file" -prop_checkBashisms51= verifyNot checkBashisms "#!/bin/sh\ncmd 2>&1" -prop_checkBashisms52= verifyNot checkBashisms "#!/bin/sh\ncmd >&2" -prop_checkBashisms53= verifyNot checkBashisms "#!/bin/sh\nprintf -- -f\n" -prop_checkBashisms54= verify checkBashisms "#!/bin/sh\nfoo+=bar" -prop_checkBashisms55= verify checkBashisms "#!/bin/sh\necho ${@%foo}" -prop_checkBashisms56= verifyNot checkBashisms "#!/bin/sh\necho ${##}" -prop_checkBashisms57= verifyNot checkBashisms "#!/bin/dash\nulimit -c 0" -prop_checkBashisms58= verify checkBashisms "#!/bin/sh\nulimit -c 0" +prop_checkBashisms10 = verify checkBashisms "echo ${var:4:12}" +prop_checkBashisms11 = verifyNot checkBashisms "echo ${var:-4}" +prop_checkBashisms12 = verify checkBashisms "echo ${var//foo/bar}" +prop_checkBashisms13 = verify checkBashisms "exec -c env" +prop_checkBashisms14 = verify checkBashisms "echo -n \"Foo: \"" +prop_checkBashisms15 = verify checkBashisms "let n++" +prop_checkBashisms16 = verify checkBashisms "echo $RANDOM" +prop_checkBashisms17 = verify checkBashisms "echo $((RANDOM%6+1))" +prop_checkBashisms18 = verify checkBashisms "foo &> /dev/null" +prop_checkBashisms19 = verify checkBashisms "foo > file*.txt" +prop_checkBashisms20 = verify checkBashisms "read -ra foo" +prop_checkBashisms21 = verify checkBashisms "[ -a foo ]" +prop_checkBashisms22 = verifyNot checkBashisms "[ foo -a bar ]" +prop_checkBashisms23 = verify checkBashisms "trap mything ERR INT" +prop_checkBashisms24 = verifyNot checkBashisms "trap mything INT TERM" +prop_checkBashisms25 = verify checkBashisms "cat < /dev/tcp/host/123" +prop_checkBashisms26 = verify checkBashisms "trap mything ERR SIGTERM" +prop_checkBashisms27 = verify checkBashisms "echo *[^0-9]*" +prop_checkBashisms28 = verify checkBashisms "exec {n}>&2" +prop_checkBashisms29 = verify checkBashisms "echo ${!var}" +prop_checkBashisms30 = verify checkBashisms "printf -v '%s' \"$1\"" +prop_checkBashisms31 = verify checkBashisms "printf '%q' \"$1\"" +prop_checkBashisms32 = verifyNot checkBashisms "#!/bin/dash\n[ foo -nt bar ]" +prop_checkBashisms33 = verify checkBashisms "#!/bin/sh\necho -n foo" +prop_checkBashisms34 = verifyNot checkBashisms "#!/bin/dash\necho -n foo" +prop_checkBashisms35 = verifyNot checkBashisms "#!/bin/dash\nlocal foo" +prop_checkBashisms36 = verifyNot checkBashisms "#!/bin/dash\nread -p foo -r bar" +prop_checkBashisms37 = verifyNot checkBashisms "HOSTNAME=foo; echo $HOSTNAME" +prop_checkBashisms38 = verify checkBashisms "RANDOM=9; echo $RANDOM" +prop_checkBashisms39 = verify checkBashisms "foo-bar() { true; }" +prop_checkBashisms40 = verify checkBashisms "echo $(/dev/null" +prop_checkBashisms48 = verifyNot checkBashisms "#!/bin/sh\necho $LINENO" +prop_checkBashisms49 = verify checkBashisms "#!/bin/dash\necho $MACHTYPE" +prop_checkBashisms50 = verify checkBashisms "#!/bin/sh\ncmd >& file" +prop_checkBashisms51 = verifyNot checkBashisms "#!/bin/sh\ncmd 2>&1" +prop_checkBashisms52 = verifyNot checkBashisms "#!/bin/sh\ncmd >&2" +prop_checkBashisms53 = verifyNot checkBashisms "#!/bin/sh\nprintf -- -f\n" +prop_checkBashisms54 = verify checkBashisms "#!/bin/sh\nfoo+=bar" +prop_checkBashisms55 = verify checkBashisms "#!/bin/sh\necho ${@%foo}" +prop_checkBashisms56 = verifyNot checkBashisms "#!/bin/sh\necho ${##}" +prop_checkBashisms57 = verifyNot checkBashisms "#!/bin/dash\nulimit -c 0" +prop_checkBashisms58 = verify checkBashisms "#!/bin/sh\nulimit -c 0" prop_checkBashisms59 = verify checkBashisms "#!/bin/sh\njobs -s" prop_checkBashisms60 = verifyNot checkBashisms "#!/bin/sh\njobs -p" prop_checkBashisms61 = verifyNot checkBashisms "#!/bin/sh\njobs -lp" @@ -441,9 +441,9 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do _ -> False prop_checkEchoSed1 = verify checkEchoSed "FOO=$(echo \"$cow\" | sed 's/foo/bar/g')" -prop_checkEchoSed1b= verify checkEchoSed "FOO=$(sed 's/foo/bar/g' <<< \"$cow\")" +prop_checkEchoSed1b = verify checkEchoSed "FOO=$(sed 's/foo/bar/g' <<< \"$cow\")" prop_checkEchoSed2 = verify checkEchoSed "rm $(echo $cow | sed -e 's,foo,bar,')" -prop_checkEchoSed2b= verify checkEchoSed "rm $(sed -e 's,foo,bar,' <<< $cow)" +prop_checkEchoSed2b = verify checkEchoSed "rm $(sed -e 's,foo,bar,' <<< $cow)" checkEchoSed = ForShell [Bash, Ksh] f where f (T_Redirecting id lefts r) = @@ -529,11 +529,11 @@ checkMultiDimensionalArrays = ForShell [Bash] f isMultiDim l = getBracedModifier (concat $ oversimplify l) `matches` re prop_checkPS11 = verify checkPS1Assignments "PS1='\\033[1;35m\\$ '" -prop_checkPS11a= verify checkPS1Assignments "export PS1='\\033[1;35m\\$ '" +prop_checkPS11a = verify checkPS1Assignments "export PS1='\\033[1;35m\\$ '" prop_checkPSf2 = verify checkPS1Assignments "PS1='\\h \\e[0m\\$ '" prop_checkPS13 = verify checkPS1Assignments "PS1=$'\\x1b[c '" prop_checkPS14 = verify checkPS1Assignments "PS1=$'\\e[3m; '" -prop_checkPS14a= verify checkPS1Assignments "export PS1=$'\\e[3m; '" +prop_checkPS14a = verify checkPS1Assignments "export PS1=$'\\e[3m; '" prop_checkPS15 = verifyNot checkPS1Assignments "PS1='\\[\\033[1;35m\\]\\$ '" prop_checkPS16 = verifyNot checkPS1Assignments "PS1='\\[\\e1m\\e[1m\\]\\$ '" prop_checkPS17 = verifyNot checkPS1Assignments "PS1='e033x1B'" diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 9f9241c..e6a2999 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -719,20 +719,20 @@ prop_a6 = isOk readArithmeticContents " 1 | 2 ||3|4" prop_a7 = isOk readArithmeticContents "3*2**10" prop_a8 = isOk readArithmeticContents "3" prop_a9 = isOk readArithmeticContents "a^!-b" -prop_a10= isOk readArithmeticContents "! $?" -prop_a11= isOk readArithmeticContents "10#08 * 16#f" -prop_a12= isOk readArithmeticContents "\"$((3+2))\" + '37'" -prop_a13= isOk readArithmeticContents "foo[9*y+x]++" -prop_a14= isOk readArithmeticContents "1+`echo 2`" -prop_a15= isOk readArithmeticContents "foo[`echo foo | sed s/foo/4/g` * 3] + 4" -prop_a16= isOk readArithmeticContents "$foo$bar" -prop_a17= isOk readArithmeticContents "i<(0+(1+1))" -prop_a18= isOk readArithmeticContents "a?b:c" -prop_a19= isOk readArithmeticContents "\\\n3 +\\\n 2" -prop_a20= isOk readArithmeticContents "a ? b ? c : d : e" -prop_a21= isOk readArithmeticContents "a ? b : c ? d : e" -prop_a22= isOk readArithmeticContents "!!a" -prop_a23= isOk readArithmeticContents "~0" +prop_a10 = isOk readArithmeticContents "! $?" +prop_a11 = isOk readArithmeticContents "10#08 * 16#f" +prop_a12 = isOk readArithmeticContents "\"$((3+2))\" + '37'" +prop_a13 = isOk readArithmeticContents "foo[9*y+x]++" +prop_a14 = isOk readArithmeticContents "1+`echo 2`" +prop_a15 = isOk readArithmeticContents "foo[`echo foo | sed s/foo/4/g` * 3] + 4" +prop_a16 = isOk readArithmeticContents "$foo$bar" +prop_a17 = isOk readArithmeticContents "i<(0+(1+1))" +prop_a18 = isOk readArithmeticContents "a?b:c" +prop_a19 = isOk readArithmeticContents "\\\n3 +\\\n 2" +prop_a20 = isOk readArithmeticContents "a ? b ? c : d : e" +prop_a21 = isOk readArithmeticContents "a ? b : c ? d : e" +prop_a22 = isOk readArithmeticContents "!!a" +prop_a23 = isOk readArithmeticContents "~0" readArithmeticContents :: Monad m => SCParser m Token readArithmeticContents = readSequence @@ -925,8 +925,8 @@ prop_readCondition7 = isOk readCondition "[[ ${line} =~ ^[[:space:]]*# ]]" prop_readCondition8 = isOk readCondition "[[ $l =~ ogg|flac ]]" prop_readCondition9 = isOk readCondition "[ foo -a -f bar ]" prop_readCondition10 = isOk readCondition "[[\na == b\n||\nc == d ]]" -prop_readCondition10a= isOk readCondition "[[\na == b ||\nc == d ]]" -prop_readCondition10b= isOk readCondition "[[ a == b\n||\nc == d ]]" +prop_readCondition10a = isOk readCondition "[[\na == b ||\nc == d ]]" +prop_readCondition10b = isOk readCondition "[[ a == b\n||\nc == d ]]" prop_readCondition11 = isOk readCondition "[[ a == b ||\n c == d ]]" prop_readCondition12 = isWarning readCondition "[ a == b \n -o c == d ]" prop_readCondition13 = isOk readCondition "[[ foo =~ ^fo{1,3}$ ]]" @@ -1701,9 +1701,9 @@ readDollarBraced = called "parameter expansion" $ do id <- endSpan start return $ T_DollarBraced id True word -prop_readDollarExpansion1= isOk readDollarExpansion "$(echo foo; ls\n)" -prop_readDollarExpansion2= isOk readDollarExpansion "$( )" -prop_readDollarExpansion3= isOk readDollarExpansion "$( command \n#comment \n)" +prop_readDollarExpansion1 = isOk readDollarExpansion "$(echo foo; ls\n)" +prop_readDollarExpansion2 = isOk readDollarExpansion "$( )" +prop_readDollarExpansion3 = isOk readDollarExpansion "$( command \n#comment \n)" readDollarExpansion = called "command expansion" $ do start <- startSpan try (string "$(") @@ -1795,17 +1795,17 @@ prop_readHereDoc6 = isOk readScript "cat << foo\\ bar\ncow\nfoo bar" prop_readHereDoc7 = isOk readScript "cat << foo\n\\$(f ())\nfoo" prop_readHereDoc8 = isOk readScript "cat <>bar\netc\nfoo" prop_readHereDoc9 = isOk readScript "if true; then cat << foo; fi\nbar\nfoo\n" -prop_readHereDoc10= isOk readScript "if true; then cat << foo << bar; fi\nfoo\nbar\n" -prop_readHereDoc11= isOk readScript "cat << foo $(\nfoo\n)lol\nfoo\n" -prop_readHereDoc12= isOk readScript "cat << foo|cat\nbar\nfoo" -prop_readHereDoc13= isOk readScript "cat <<'#!'\nHello World\n#!\necho Done" -prop_readHereDoc14= isWarning readScript "cat << foo\nbar\nfoo \n" -prop_readHereDoc15= isWarning readScript "cat < Date: Fri, 22 Jul 2022 17:06:24 -0700 Subject: [PATCH 248/459] Omit SC3021 about `>& file` unless definitely non-numeric (fixes #2520) --- src/ShellCheck/Checks/ShellSupport.hs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index c12da2d..30a19b9 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -135,6 +135,8 @@ prop_checkBashisms49 = verify checkBashisms "#!/bin/dash\necho $MACHTYPE" prop_checkBashisms50 = verify checkBashisms "#!/bin/sh\ncmd >& file" prop_checkBashisms51 = verifyNot checkBashisms "#!/bin/sh\ncmd 2>&1" prop_checkBashisms52 = verifyNot checkBashisms "#!/bin/sh\ncmd >&2" +prop_checkBashisms52b = verifyNot checkBashisms "#!/bin/sh\ncmd >& $var" +prop_checkBashisms52c = verify checkBashisms "#!/bin/sh\ncmd >& $dir/$var" prop_checkBashisms53 = verifyNot checkBashisms "#!/bin/sh\nprintf -- -f\n" prop_checkBashisms54 = verify checkBashisms "#!/bin/sh\nfoo+=bar" prop_checkBashisms55 = verify checkBashisms "#!/bin/sh\necho ${@%foo}" @@ -225,7 +227,8 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do warnMsg id 3018 $ filter (/= '|') op ++ " is" bashism (TA_Binary id "**" _ _) = warnMsg id 3019 "exponentials are" bashism (T_FdRedirect id "&" (T_IoFile _ (T_Greater _) _)) = warnMsg id 3020 "&> is" - bashism (T_FdRedirect id "" (T_IoFile _ (T_GREATAND _) _)) = warnMsg id 3021 ">& is" + bashism (T_FdRedirect id "" (T_IoFile _ (T_GREATAND _) file)) = + unless (all isDigit $ onlyLiteralString file) $ warnMsg id 3021 ">& filename (as opposed to >& fd) is" bashism (T_FdRedirect id ('{':_) _) = warnMsg id 3022 "named file descriptors are" bashism (T_FdRedirect id num _) | all isDigit num && length num > 1 = warnMsg id 3023 "FDs outside 0-9 are" From b261ec24f9ebcb911e7ff4264e5ea7d36cc93f59 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Fri, 22 Jul 2022 20:16:01 -0700 Subject: [PATCH 249/459] Include exit codes in DFA (ref #2541) --- src/ShellCheck/CFG.hs | 8 ++- src/ShellCheck/CFGAnalysis.hs | 92 +++++++++++++++++++++++++++-------- 2 files changed, 77 insertions(+), 23 deletions(-) diff --git a/src/ShellCheck/CFG.hs b/src/ShellCheck/CFG.hs index 4906d80..1085d8f 100644 --- a/src/ShellCheck/CFG.hs +++ b/src/ShellCheck/CFG.hs @@ -651,7 +651,10 @@ build t = do pg <- wordToExactPseudoGlob c return $ pg `pseudoGlobIsSuperSetof` [PGMany] - T_Condition _ _ op -> build op + T_Condition id _ op -> do + cond <- build op + status <- newNodeRange $ CFSetExitCode id + linkRange cond status T_CoProc id maybeName t -> do let name = fromMaybe "COPROC" maybeName @@ -798,7 +801,8 @@ build t = do start <- newStructuralNode hasLastpipe <- reader $ cfLastpipe . cfParameters (leading, last) <- buildPipe hasLastpipe cmds - end <- newStructuralNode + -- Ideally we'd let this exit code be that of the last command in the pipeline but ok + end <- newNodeRange $ CFSetExitCode id mapM_ (linkRange start) leading mapM_ (\c -> linkRangeAs CFEFalseFlow c end) leading diff --git a/src/ShellCheck/CFGAnalysis.hs b/src/ShellCheck/CFGAnalysis.hs index daade43..893c34a 100644 --- a/src/ShellCheck/CFGAnalysis.hs +++ b/src/ShellCheck/CFGAnalysis.hs @@ -104,11 +104,29 @@ data CFGAnalysis = CFGAnalysis { -- The program state we expose externally data ProgramState = ProgramState { --- internalState :: InternalState, -- For debugging + -- internalState :: InternalState, -- For debugging variablesInScope :: M.Map String VariableState, + exitCodes :: S.Set Id, stateIsReachable :: Bool } deriving (Show, Eq, Generic, NFData) +internalToExternal :: InternalState -> ProgramState +internalToExternal s = + ProgramState { + -- Censor the literal value to avoid introducing dependencies on it. It's just for debugging. + variablesInScope = M.map censor flatVars, + -- internalState = s, -- For debugging + exitCodes = fromMaybe S.empty $ sExitCodes s, + stateIsReachable = fromMaybe True $ sIsReachable s + } + where + censor s = s { + variableValue = (variableValue s) { + literalValue = Nothing + } + } + flatVars = M.unionsWith (\_ last -> last) $ map mapStorage [sGlobalValues s, sLocalValues s, sPrefixValues s] + -- Conveniently get the state before a token id getIncomingState :: CFGAnalysis -> Id -> Maybe ProgramState getIncomingState analysis id = do @@ -130,6 +148,7 @@ data InternalState = InternalState { sLocalValues :: VersionedMap String VariableState, sPrefixValues :: VersionedMap String VariableState, sFunctionTargets :: VersionedMap String FunctionValue, + sExitCodes :: Maybe (S.Set Id), sIsReachable :: Maybe Bool } deriving (Show, Generic, NFData) @@ -139,6 +158,7 @@ newInternalState = InternalState { sLocalValues = vmEmpty, sPrefixValues = vmEmpty, sFunctionTargets = vmEmpty, + sExitCodes = Nothing, sIsReachable = Nothing } @@ -196,31 +216,25 @@ removeProperties props state = state { variableProperties = S.map (\s -> S.difference s props) $ variableProperties state } -internalToExternal :: InternalState -> ProgramState -internalToExternal s = - ProgramState { - -- Censor the literal value to avoid introducing dependencies on it. It's just for debugging. - variablesInScope = M.map censor flatVars, - -- internalState = s, -- For debugging - stateIsReachable = fromMaybe True $ sIsReachable s - } - where - censor s = s { - variableValue = (variableValue s) { - literalValue = Nothing - } - } - flatVars = M.unionsWith (\_ last -> last) $ map mapStorage [sGlobalValues s, sLocalValues s, sPrefixValues s] +setExitCode id = setExitCodes (S.singleton id) +setExitCodes set state = modified state { + sExitCodes = Just $ set +} -- Dependencies on values, e.g. "if there is a global variable named 'foo' without spaces" -- This is used to see if the DFA of a function would result in the same state, so anything -- that affects DFA must be tracked. data StateDependency = + -- Complete variable state DepState Scope String VariableState + -- Only variable properties (we need properties but not values for x=1) | DepProperties Scope String VariableProperties + -- Function definition | DepFunction String (S.Set FunctionDefinition) -- Whether invoking the node would result in recursion (i.e., is the function on the stack?) | DepIsRecursive Node Bool + -- The set of commands that could have provided the exit code $? + | DepExitCodes (S.Set Id) deriving (Show, Eq, Ord, Generic, NFData) -- A function definition, or lack thereof @@ -242,6 +256,7 @@ depsToState set = foldl insert newInternalState $ S.toList set -- State includes properties and more, so don't overwrite a state with properties DepProperties scope name props -> insertIn False scope name unknownVariableState { variableProperties = props } state DepIsRecursive _ _ -> state + DepExitCodes s -> setExitCodes s state insertIn overwrite scope name val state = let @@ -400,6 +415,7 @@ patchState base diff = sLocalValues = vmPatch (sLocalValues base) (sLocalValues diff), sPrefixValues = vmPatch (sPrefixValues base) (sPrefixValues diff), sFunctionTargets = vmPatch (sFunctionTargets base) (sFunctionTargets diff), + sExitCodes = sExitCodes diff `mplus` sExitCodes base, sIsReachable = sIsReachable diff `mplus` sIsReachable base } @@ -444,12 +460,14 @@ mergeState ctx a b = do locals <- mergeMaps ctx mergeVariableState readVariable (sLocalValues a) (sLocalValues b) prefix <- mergeMaps ctx mergeVariableState readVariable (sPrefixValues a) (sPrefixValues b) funcs <- mergeMaps ctx S.union readFunction (sFunctionTargets a) (sFunctionTargets b) + exitCodes <- mergeMaybes ctx S.union readExitCodes (sExitCodes a) (sExitCodes b) return $ InternalState { sVersion = -1, sGlobalValues = globals, sLocalValues = locals, sPrefixValues = prefix, sFunctionTargets = funcs, + sExitCodes = exitCodes, sIsReachable = liftM2 (&&) (sIsReachable a) (sIsReachable b) } @@ -493,6 +511,18 @@ mergeMaps ctx merger reader a b = nv1 <- reader ctx k2 f ((k2, merger nv1 v2):l) l1 rest2 +-- Merge two Maybes, like mergeMaps for a single element +mergeMaybes ctx merger reader a b = + case (a, b) of + (Nothing, Nothing) -> return Nothing + (Just v1, Nothing) -> single v1 + (Nothing, Just v2) -> single v2 + (Just v1, Just v2) -> return $ Just $ merger v1 v2 + where + single val = do + result <- merger val <$> reader ctx + return $ Just result + vmFromMap ctx map = return $ VersionedMap { mapVersion = -1, mapStorage = map @@ -708,6 +738,12 @@ readFunction ctx name = lookupStack get dep def ctx name writeFunction ctx name val = do modifySTRef (cOutput ctx) $ insertFunction name $ S.singleton val +readExitCodes ctx = lookupStack get dep def ctx () + where + get s () = sExitCodes s + def = S.empty + dep () v = DepExitCodes v + -- Look up each state on the stack until a value is found (or the default is used), -- then add this value as a StateDependency. lookupStack' :: forall s k v. @@ -872,13 +908,13 @@ transfer ctx label = CFExecuteCommand cmd -> transferCommand ctx cmd CFExecuteSubshell reason entry exit -> transferSubshell ctx reason entry exit CFApplyEffects effects -> mapM_ (\(IdTagged _ f) -> transferEffect ctx f) effects + CFSetExitCode id -> transferExitCode ctx id CFUnresolvedExit -> patchOutputM ctx unreachableState CFUnreachable -> patchOutputM ctx unreachableState -- TODO CFSetBackgroundPid _ -> return () - CFSetExitCode _ -> return () CFDropPrefixAssignments {} -> modifySTRef (cOutput ctx) $ \c -> modified c { sPrefixValues = vmEmpty } -- _ -> error $ "Unknown " ++ show label @@ -891,8 +927,11 @@ transferSubshell ctx reason entry exit = do let cout = cOutput ctx initial <- readSTRef cout runCached ctx entry (f entry exit) + res <- readSTRef cout -- Clear subshell changes. TODO: track this to warn about modifications. - writeSTRef cout initial + writeSTRef cout $ initial { + sExitCodes = sExitCodes res + } where f entry exit ctx = do (states, frame) <- withNewStackFrame ctx entry False (flip dataflow $ entry) @@ -947,6 +986,8 @@ transferFunctionValue ctx funcVal = registerFlowResult ctx entry states deps return (deps, res) +transferExitCode ctx id = do + modifySTRef (cOutput ctx) $ setExitCode id -- Register/save the result of a dataflow of a function. -- At the end, all the different values from different flows are merged together. @@ -1001,8 +1042,10 @@ getCache ctx node = do -- Transfer a single CFEffect to the output state. transferEffect ctx effect = case effect of - CFReadVariable name -> do - void $ readVariable ctx name + CFReadVariable name -> + case name of + "?" -> void $ readExitCodes ctx + _ -> void $ readVariable ctx name CFWriteVariable name value -> do val <- cfValueToVariableValue ctx value updateVariableValue ctx name val @@ -1235,7 +1278,14 @@ analyzeControlFlow params t = -- (it's probably not actually dead, just used by a script that sources ours) let declaredFunctions = getFunctionTargets exitState let uninvoked = M.difference declaredFunctions invokedNodes - analyzeStragglers ctx exitState uninvoked + + let stragglerInput = + exitState { + -- We don't want `die() { exit $?; }; echo "Sourced"` to assume $? is always echo + sExitCodes = Nothing + } + + analyzeStragglers ctx stragglerInput uninvoked -- Now round up all the states from all data flows -- (FIXME: this excludes functions that were defined in straggling functions) From f7857028f7a019b47c8a5294dd8d4182e5bed864 Mon Sep 17 00:00:00 2001 From: ygeyzel Date: Sat, 23 Jul 2022 19:24:16 +0300 Subject: [PATCH 250/459] Add escape characters to SC2028: \a, \b, \e, \f, \v, \\, \', \OOO, \xHH --- src/ShellCheck/Checks/Commands.hs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 4bae17e..67c3c48 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -481,9 +481,16 @@ prop_checkUnusedEchoEscapes2 = verifyNot checkUnusedEchoEscapes "echo -e 'foi\\n prop_checkUnusedEchoEscapes3 = verify checkUnusedEchoEscapes "echo \"n:\\t42\"" prop_checkUnusedEchoEscapes4 = verifyNot checkUnusedEchoEscapes "echo lol" prop_checkUnusedEchoEscapes5 = verifyNot checkUnusedEchoEscapes "echo -n -e '\n'" +prop_checkUnusedEchoEscapes6 = verify checkUnusedEchoEscapes "echo '\\506'" +prop_checkUnusedEchoEscapes7 = verify checkUnusedEchoEscapes "echo '\\5a'" +prop_checkUnusedEchoEscapes8 = verifyNot checkUnusedEchoEscapes "echo '\\8a'" +prop_checkUnusedEchoEscapes9 = verifyNot checkUnusedEchoEscapes "echo '\\d5a'" +prop_checkUnusedEchoEscapes10 = verify checkUnusedEchoEscapes "echo '\\x4a'" +prop_checkUnusedEchoEscapes11 = verify checkUnusedEchoEscapes "echo '\\xat'" +prop_checkUnusedEchoEscapes12 = verifyNot checkUnusedEchoEscapes "echo '\\xth'" checkUnusedEchoEscapes = CommandCheck (Basename "echo") f where - hasEscapes = mkRegex "\\\\[rnt]" + hasEscapes = mkRegex "\\\\([rntabefv\\']|[0-7]{1,3}|x([0-9]|[A-F]|[a-f]){1,2})" f cmd = whenShell [Sh, Bash, Ksh] $ unless (cmd `hasFlag` "e") $ From 5cf6e01ce902421065d8eee24c7a4a10d842efa9 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 23 Jul 2022 09:38:58 -0700 Subject: [PATCH 251/459] Warn when $? refers to echo or condition (ref #2541) --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 37 +++++++++++++++++++++++++++++++++++ src/ShellCheck/AnalyzerLib.hs | 3 +++ src/ShellCheck/CFGAnalysis.hs | 1 + 4 files changed, 42 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index dce3e27..5ff01c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ - SC2316: Warn about 'local readonly foo' and similar (thanks, patrickxia!) - SC2317: Warn about unreachable commands - SC2318: Warn about backreferences in 'declare x=1 y=$x' +- SC2319/SC2320: Warn when $? refers to echo/printf/[ ]/[[ ]]/test ### Fixed - SC2086: Now uses DFA to make more accurate predictions about values diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 1429e1b..07bf25b 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -205,6 +205,7 @@ nodeChecks = [ ,checkBatsTestDoesNotUseNegation ,checkCommandIsUnreachable ,checkSpacefulnessCfg + ,checkOverwrittenExitCode ] optionalChecks = map fst optionalTreeChecks @@ -4876,5 +4877,41 @@ checkCommandIsUnreachable params t = _ -> return () where id = getId t + +prop_checkOverwrittenExitCode1 = verify checkOverwrittenExitCode "x; [ $? -eq 1 ] || [ $? -eq 2 ]" +prop_checkOverwrittenExitCode2 = verifyNot checkOverwrittenExitCode "x; [ $? -eq 1 ]" +prop_checkOverwrittenExitCode3 = verify checkOverwrittenExitCode "x; echo \"Exit is $?\"; [ $? -eq 0 ]" +prop_checkOverwrittenExitCode4 = verifyNot checkOverwrittenExitCode "x; [ $? -eq 0 ]" +checkOverwrittenExitCode params t = + case t of + T_DollarBraced id _ val | getLiteralString val == Just "?" -> check id + _ -> return () + where + check id = sequence_ $ do + state <- CF.getIncomingState (cfgAnalysis params) id + let exitCodeIds = CF.exitCodes state + guard . not $ S.null exitCodeIds + + let idToToken = idMap params + exitCodeTokens <- sequence $ map (\k -> Map.lookup k idToToken) $ S.toList exitCodeIds + return $ do + when (all isCondition exitCodeTokens) $ + warn id 2319 "This $? refers to a condition, not a command. Assign to a variable to avoid it being overwritten." + when (all isPrinting exitCodeTokens) $ + warn id 2320 "This $? refers to echo/printf, not a previous command. Assign to variable to avoid it being overwritten." + + isCondition t = + case t of + T_Condition {} -> True + T_SimpleCommand {} -> getCommandName t == Just "test" + _ -> False + + isPrinting t = + case getCommandBasename t of + Just "echo" -> True + Just "printf" -> True + _ -> False + + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index e998f2c..88da89e 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -89,6 +89,8 @@ data Parameters = Parameters { hasPipefail :: Bool, -- A linear (bad) analysis of data flow variableFlow :: [StackData], + -- A map from Id to Token + idMap :: Map.Map Id Token, -- A map from Id to parent Token parentMap :: Map.Map Id Token, -- The shell type, such as Bash or Ksh @@ -218,6 +220,7 @@ makeParameters spec = params Sh -> True Ksh -> containsPipefail root, shellTypeSpecified = isJust (asShellType spec) || isJust (asFallbackShell spec), + idMap = getTokenMap root, parentMap = getParentTree root, variableFlow = getVariableFlow params root, tokenPositions = asTokenPositions spec, diff --git a/src/ShellCheck/CFGAnalysis.hs b/src/ShellCheck/CFGAnalysis.hs index 893c34a..bb90860 100644 --- a/src/ShellCheck/CFGAnalysis.hs +++ b/src/ShellCheck/CFGAnalysis.hs @@ -804,6 +804,7 @@ fulfillsDependency ctx entry dep = -- it won't be found by the normal check. DepIsRecursive node val | node == entry -> return True DepIsRecursive node val -> return $ val == any (\f -> entryPoint f == node) (cStack ctx) + DepExitCodes val -> (== val) <$> peekStack (\s k -> sExitCodes s) S.empty ctx () -- _ -> error $ "Unknown dep " ++ show dep where peek scope = peekStack getVariableWithScope $ if scope == GlobalScope then (unknownVariableState, GlobalScope) else (unsetVariableState, LocalScope) From ea4e0091c7b403ace135eddf1e508ff2fc0f783f Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 23 Jul 2022 15:38:42 -0700 Subject: [PATCH 252/459] Additionally pluralize 'arguments' in SC2183 --- src/ShellCheck/Checks/Commands.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 8bca2ba..e97ecd6 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -706,8 +706,8 @@ checkPrintfVar = CommandCheck (Exactly "printf") (f . arguments) where return () -- Great: a suitable number of arguments | otherwise -> warn (getId format) 2183 $ - "This format string has " ++ show formatCount ++ " " ++ (pluraliseIfMany "variable" formatCount) ++ - ", but is passed " ++ show argCount ++ " arguments." + "This format string has " ++ show formatCount ++ " " ++ pluraliseIfMany "variable" formatCount ++ + ", but is passed " ++ show argCount ++ pluraliseIfMany " argument" argCount ++ "." unless ('%' `elem` concat (oversimplify format) || isLiteral format) $ info (getId format) 2059 From 30bb0e0093172b93fecb53dca9d5d657962d8c0a Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 23 Jul 2022 20:10:58 -0700 Subject: [PATCH 253/459] SC2321: Warn about redundant $(()) in arr[$((i))]=x (ref: #1666) --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ff01c3..669579f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - SC2317: Warn about unreachable commands - SC2318: Warn about backreferences in 'declare x=1 y=$x' - SC2319/SC2320: Warn when $? refers to echo/printf/[ ]/[[ ]]/test +- SC2321: Suggest removing $((..)) in array[$((idx))]=val ### Fixed - SC2086: Now uses DFA to make more accurate predictions about values diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 07bf25b..e92f3ff 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -206,6 +206,7 @@ nodeChecks = [ ,checkCommandIsUnreachable ,checkSpacefulnessCfg ,checkOverwrittenExitCode + ,checkUnnecessaryArithmeticExpansionIndex ] optionalChecks = map fst optionalTreeChecks @@ -4913,5 +4914,22 @@ checkOverwrittenExitCode params t = _ -> False +prop_checkUnnecessaryArithmeticExpansionIndex1 = verify checkUnnecessaryArithmeticExpansionIndex "a[$((1+1))]=n" +prop_checkUnnecessaryArithmeticExpansionIndex2 = verifyNot checkUnnecessaryArithmeticExpansionIndex "a[1+1]=n" +prop_checkUnnecessaryArithmeticExpansionIndex3 = verifyNot checkUnnecessaryArithmeticExpansionIndex "a[$(echo $((1+1)))]=n" +checkUnnecessaryArithmeticExpansionIndex params t = + case t of + T_Assignment _ mode var [TA_Sequence _ [ TA_Expansion _ [expansion@(T_DollarArithmetic id _)]]] val -> + styleWithFix id 2321 "Array indices are already arithmetic contexts. Prefer removing the $(( and ))." $ fix id + _ -> return () + + where + fix id = + fixWith [ + replaceStart id params 3 "", -- Remove "$((" + replaceEnd id params 2 "" -- Remove "))" + ] + + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) From 52dac51cd4d919667e3a2d3d3feec1d516464392 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 24 Jul 2022 14:06:01 -0700 Subject: [PATCH 254/459] SC2323: Warn about redundant parens in a[(x+1)] and $(( ((x)) )) (ref: #1666) --- CHANGELOG.md | 2 ++ src/ShellCheck/AST.hs | 4 +++- src/ShellCheck/Analytics.hs | 33 +++++++++++++++++++++++++++++++++ src/ShellCheck/CFG.hs | 1 + src/ShellCheck/Parser.hs | 4 +++- 5 files changed, 42 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 669579f..cef16f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ - SC2318: Warn about backreferences in 'declare x=1 y=$x' - SC2319/SC2320: Warn when $? refers to echo/printf/[ ]/[[ ]]/test - SC2321: Suggest removing $((..)) in array[$((idx))]=val +- SC2322: Suggest collapsing double parentheses in arithmetic contexts +- SC2323: Suggest removing wrapping parentheses in a[(x+1)]=val ### Fixed - SC2086: Now uses DFA to make more accurate predictions about values diff --git a/src/ShellCheck/AST.hs b/src/ShellCheck/AST.hs index 2cd2f6f..ca5007a 100644 --- a/src/ShellCheck/AST.hs +++ b/src/ShellCheck/AST.hs @@ -45,6 +45,7 @@ data InnerToken t = | Inner_TA_Variable String [t] | Inner_TA_Expansion [t] | Inner_TA_Sequence [t] + | Inner_TA_Parenthesis t | Inner_TA_Trinary t t t | Inner_TA_Unary String t | Inner_TC_And ConditionType String t t @@ -204,6 +205,7 @@ pattern T_Annotation id anns t = OuterToken id (Inner_T_Annotation anns t) pattern T_Arithmetic id c = OuterToken id (Inner_T_Arithmetic c) pattern T_Array id t = OuterToken id (Inner_T_Array t) pattern TA_Sequence id l = OuterToken id (Inner_TA_Sequence l) +pattern TA_Parentesis id t = OuterToken id (Inner_TA_Parenthesis t) pattern T_Assignment id mode var indices value = OuterToken id (Inner_T_Assignment mode var indices value) pattern TA_Trinary id t1 t2 t3 = OuterToken id (Inner_TA_Trinary t1 t2 t3) pattern TA_Unary id op t1 = OuterToken id (Inner_TA_Unary op t1) @@ -256,7 +258,7 @@ pattern T_Subshell id l = OuterToken id (Inner_T_Subshell l) pattern T_UntilExpression id c l = OuterToken id (Inner_T_UntilExpression c l) pattern T_WhileExpression id c l = OuterToken id (Inner_T_WhileExpression c l) -{-# COMPLETE T_AND_IF, T_Bang, T_Case, TC_Empty, T_CLOBBER, T_DGREAT, T_DLESS, T_DLESSDASH, T_Do, T_DollarSingleQuoted, T_Done, T_DSEMI, T_Elif, T_Else, T_EOF, T_Esac, T_Fi, T_For, T_Glob, T_GREATAND, T_Greater, T_If, T_In, T_Lbrace, T_Less, T_LESSAND, T_LESSGREAT, T_Literal, T_Lparen, T_NEWLINE, T_OR_IF, T_ParamSubSpecialChar, T_Pipe, T_Rbrace, T_Rparen, T_Select, T_Semi, T_SingleQuoted, T_Then, T_UnparsedIndex, T_Until, T_While, TA_Assignment, TA_Binary, TA_Expansion, T_AndIf, T_Annotation, T_Arithmetic, T_Array, TA_Sequence, T_Assignment, TA_Trinary, TA_Unary, TA_Variable, T_Backgrounded, T_Backticked, T_Banged, T_BatsTest, T_BraceExpansion, T_BraceGroup, TC_And, T_CaseExpression, TC_Binary, TC_Group, TC_Nullary, T_Condition, T_CoProcBody, T_CoProc, TC_Or, TC_Unary, T_DollarArithmetic, T_DollarBraceCommandExpansion, T_DollarBraced, T_DollarBracket, T_DollarDoubleQuoted, T_DollarExpansion, T_DoubleQuoted, T_Extglob, T_FdRedirect, T_ForArithmetic, T_ForIn, T_Function, T_HereDoc, T_HereString, T_IfExpression, T_Include, T_IndexedElement, T_IoDuplicate, T_IoFile, T_NormalWord, T_OrIf, T_Pipeline, T_ProcSub, T_Redirecting, T_Script, T_SelectIn, T_SimpleCommand, T_SourceCommand, T_Subshell, T_UntilExpression, T_WhileExpression #-} +{-# COMPLETE T_AND_IF, T_Bang, T_Case, TC_Empty, T_CLOBBER, T_DGREAT, T_DLESS, T_DLESSDASH, T_Do, T_DollarSingleQuoted, T_Done, T_DSEMI, T_Elif, T_Else, T_EOF, T_Esac, T_Fi, T_For, T_Glob, T_GREATAND, T_Greater, T_If, T_In, T_Lbrace, T_Less, T_LESSAND, T_LESSGREAT, T_Literal, T_Lparen, T_NEWLINE, T_OR_IF, T_ParamSubSpecialChar, T_Pipe, T_Rbrace, T_Rparen, T_Select, T_Semi, T_SingleQuoted, T_Then, T_UnparsedIndex, T_Until, T_While, TA_Assignment, TA_Binary, TA_Expansion, T_AndIf, T_Annotation, T_Arithmetic, T_Array, TA_Sequence, TA_Parentesis, T_Assignment, TA_Trinary, TA_Unary, TA_Variable, T_Backgrounded, T_Backticked, T_Banged, T_BatsTest, T_BraceExpansion, T_BraceGroup, TC_And, T_CaseExpression, TC_Binary, TC_Group, TC_Nullary, T_Condition, T_CoProcBody, T_CoProc, TC_Or, TC_Unary, T_DollarArithmetic, T_DollarBraceCommandExpansion, T_DollarBraced, T_DollarBracket, T_DollarDoubleQuoted, T_DollarExpansion, T_DoubleQuoted, T_Extglob, T_FdRedirect, T_ForArithmetic, T_ForIn, T_Function, T_HereDoc, T_HereString, T_IfExpression, T_Include, T_IndexedElement, T_IoDuplicate, T_IoFile, T_NormalWord, T_OrIf, T_Pipeline, T_ProcSub, T_Redirecting, T_Script, T_SelectIn, T_SimpleCommand, T_SourceCommand, T_Subshell, T_UntilExpression, T_WhileExpression #-} instance Eq Token where OuterToken _ a == OuterToken _ b = a == b diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index e92f3ff..eed2d25 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -207,6 +207,7 @@ nodeChecks = [ ,checkSpacefulnessCfg ,checkOverwrittenExitCode ,checkUnnecessaryArithmeticExpansionIndex + ,checkUnnecessaryParens ] optionalChecks = map fst optionalTreeChecks @@ -3280,6 +3281,7 @@ checkReturnAgainstZero params token = _:next@(TA_Unary _ "!" _):_ -> isOnlyTestInCommand next _:next@(TC_Group {}):_ -> isOnlyTestInCommand next _:next@(TA_Sequence _ [_]):_ -> isOnlyTestInCommand next + _:next@(TA_Parentesis _ _):_ -> isOnlyTestInCommand next _ -> False -- TODO: Do better $? tracking and filter on whether @@ -4931,5 +4933,36 @@ checkUnnecessaryArithmeticExpansionIndex params t = ] +prop_checkUnnecessaryParens1 = verify checkUnnecessaryParens "echo $(( ((1+1)) ))" +prop_checkUnnecessaryParens2 = verify checkUnnecessaryParens "x[((1+1))+1]=1" +prop_checkUnnecessaryParens3 = verify checkUnnecessaryParens "x[(1+1)]=1" +prop_checkUnnecessaryParens4 = verify checkUnnecessaryParens "$(( (x) ))" +prop_checkUnnecessaryParens5 = verify checkUnnecessaryParens "(( (x) ))" +prop_checkUnnecessaryParens6 = verifyNot checkUnnecessaryParens "x[(1+1)+1]=1" +prop_checkUnnecessaryParens7 = verifyNot checkUnnecessaryParens "(( (1*1)+1 ))" +prop_checkUnnecessaryParens8 = verifyNot checkUnnecessaryParens "(( (1)+1 ))" +checkUnnecessaryParens params t = + case t of + T_DollarArithmetic _ t -> checkLeading "$(( (x) )) is the same as $(( x ))" t + T_ForArithmetic _ x y z _ -> mapM_ (checkLeading "for (((x); (y); (z))) is the same as for ((x; y; z))") [x,y,z] + T_Assignment _ _ _ [t] _ -> checkLeading "a[(x)] is the same as a[x]" t + T_Arithmetic _ t -> checkLeading "(( (x) )) is the same as (( x ))" t + TA_Parentesis _ (TA_Sequence _ [ TA_Parentesis id _ ]) -> + styleWithFix id 2322 "In arithmetic contexts, ((x)) is the same as (x). Prefer only one layer of parentheses." $ fix id + _ -> return () + where + + checkLeading str t = + case t of + TA_Sequence _ [TA_Parentesis id _ ] -> styleWithFix id 2323 (str ++ ". Prefer not wrapping in additional parentheses.") $ fix id + _ -> return () + + fix id = + fixWith [ + replaceStart id params 1 "", -- Remove "(" + replaceEnd id params 1 "" -- Remove ")" + ] + + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) diff --git a/src/ShellCheck/CFG.hs b/src/ShellCheck/CFG.hs index 1085d8f..6f6d4f1 100644 --- a/src/ShellCheck/CFG.hs +++ b/src/ShellCheck/CFG.hs @@ -479,6 +479,7 @@ build t = do TA_Binary _ _ a b -> sequentially [a,b] TA_Expansion _ list -> sequentially list TA_Sequence _ list -> sequentially list + TA_Parentesis _ t -> build t TA_Trinary _ cond a b -> do condition <- build cond diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index e6a2999..0dd6621 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -821,11 +821,13 @@ readArithmeticContents = return $ TA_Expansion id pieces readGroup = do + start <- startSpan char '(' s <- readSequence char ')' + id <- endSpan start spacing - return s + return $ TA_Parentesis id s readArithTerm = readGroup <|> readVariable <|> readExpansion From 982681fc05ca8db887431691d1cc40633e20d828 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 24 Jul 2022 14:30:31 -0700 Subject: [PATCH 255/459] Add unit test to ensure SC2321 does not trigger on associative arrays --- src/ShellCheck/Analytics.hs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index eed2d25..8499d8d 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -4919,6 +4919,7 @@ checkOverwrittenExitCode params t = prop_checkUnnecessaryArithmeticExpansionIndex1 = verify checkUnnecessaryArithmeticExpansionIndex "a[$((1+1))]=n" prop_checkUnnecessaryArithmeticExpansionIndex2 = verifyNot checkUnnecessaryArithmeticExpansionIndex "a[1+1]=n" prop_checkUnnecessaryArithmeticExpansionIndex3 = verifyNot checkUnnecessaryArithmeticExpansionIndex "a[$(echo $((1+1)))]=n" +prop_checkUnnecessaryArithmeticExpansionIndex4 = verifyNot checkUnnecessaryArithmeticExpansionIndex "declare -A a; a[$((1+1))]=val" checkUnnecessaryArithmeticExpansionIndex params t = case t of T_Assignment _ mode var [TA_Sequence _ [ TA_Expansion _ [expansion@(T_DollarArithmetic id _)]]] val -> From f1148b8b41087dba8441a2c79980441522e3b23b Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 25 Jul 2022 10:00:50 -0700 Subject: [PATCH 256/459] Include postdominators in CFGResult --- src/ShellCheck/CFG.hs | 86 ++++++++++++++++++++++++++++++++--- src/ShellCheck/CFGAnalysis.hs | 5 +- 2 files changed, 84 insertions(+), 7 deletions(-) diff --git a/src/ShellCheck/CFG.hs b/src/ShellCheck/CFG.hs index 6f6d4f1..ad05e93 100644 --- a/src/ShellCheck/CFG.hs +++ b/src/ShellCheck/CFG.hs @@ -54,6 +54,8 @@ import qualified Data.Set as S import Control.Monad.RWS.Lazy import Data.Graph.Inductive.Graph import Data.Graph.Inductive.Query.DFS +import Data.Graph.Inductive.Basic +import Data.Graph.Inductive.Query.Dominators import Data.Graph.Inductive.PatriciaTree as G import Debug.Trace -- STRIP @@ -171,9 +173,11 @@ data CFGResult = CFGResult { -- Map from Id to nominal start&end node (i.e. assuming normal execution without exits) cfIdToRange :: M.Map Id (Node, Node), -- A set of all nodes belonging to an Id, recursively - cfIdToNodes :: M.Map Id (S.Set Node) + cfIdToNodes :: M.Map Id (S.Set Node), + -- A map to nodes that the given node postdominates + cfPostDominators :: M.Map Node (S.Set Node) } - deriving (Show) + deriving (Show, Generic, NFData) buildGraph :: CFGParameters -> Token -> CFGResult buildGraph params root = @@ -183,12 +187,20 @@ buildGraph params root = -- renumberTopologically $ removeUnnecessaryStructuralNodes base - in - CFGResult { + + idToRange = M.fromList mapping + isRealEdge (from, to, edge) = case edge of CFEFlow -> True; _ -> False + onlyRealEdges = filter isRealEdge edges + (_, mainExit) = fromJust $ M.lookup (getId root) idToRange + + result = CFGResult { cfGraph = mkGraph nodes edges, - cfIdToRange = M.fromList mapping, - cfIdToNodes = M.fromListWith S.union $ map (\(id, n) -> (id, S.singleton n)) association + cfIdToRange = idToRange, + cfIdToNodes = M.fromListWith S.union $ map (\(id, n) -> (id, S.singleton n)) association, + cfPostDominators = findPostDominators mainExit $ mkGraph nodes onlyRealEdges } + in + deepseq result result remapGraph :: M.Map Node Node -> CFW -> CFW remapGraph remap (nodes, edges, mapping, assoc) = @@ -1190,5 +1202,67 @@ tokenToParts t = -- Check if getLiteralString can handle it, if not it's unknown _ -> [maybe CFStringUnknown CFStringLiteral $ getLiteralString t] + +-- Change all subshell invocations to instead link directly to their contents. +-- This is used for producing dominator trees. +inlineSubshells :: CFGraph -> CFGraph +inlineSubshells graph = relinkedGraph + where + subshells = ufold find [] graph + find (incoming, node, label, outgoing) acc = + case label of + CFExecuteSubshell _ start end -> (node, label, start, end, incoming, outgoing):acc + _ -> acc + + relinkedGraph = foldl' relink graph subshells + relink graph (node, label, start, end, incoming, outgoing) = + let + -- Link CFExecuteSubshell to the CFEntryPoint + subshellToStart = (incoming, node, label, [(CFEFlow, start)]) + -- Link the subshell exit to the + endToNexts = (endIncoming, endNode, endLabel, outgoing) + (endIncoming, endNode, endLabel, _) = context graph end + in + subshellToStart & (endToNexts & graph) + +findEntryNodes :: CFGraph -> [Node] +findEntryNodes graph = ufold find [] graph + where + find (incoming, node, label, _) list = + case label of + CFEntryPoint {} | null incoming -> node:list + _ -> list + +findDominators main graph = asSetMap + where + inlined = inlineSubshells graph + entryNodes = main : findEntryNodes graph + asLists = concatMap (dom inlined) entryNodes + asSetMap = M.fromList $ map (\(node, list) -> (node, S.fromList list)) asLists + +findTerminalNodes :: CFGraph -> [Node] +findTerminalNodes graph = ufold find [] graph + where + find (_, node, label, _) list = + case label of + CFUnresolvedExit -> node:list + CFApplyEffects effects -> f effects list + _ -> list + + f [] list = list + f (IdTagged _ (CFDefineFunction _ id start end):rest) list = f rest (end:list) + f (_:rest) list = f rest list + +findPostDominators :: Node -> CFGraph -> M.Map Node (S.Set Node) +findPostDominators mainexit graph = asSetMap + where + inlined = inlineSubshells graph + terminals = findTerminalNodes inlined + (incoming, _, label, outgoing) = context graph mainexit + withExitEdges = (incoming ++ map (\c -> (CFEFlow, c)) terminals, mainexit, label, outgoing) & inlined + reversed = grev withExitEdges + postDoms = dom reversed mainexit + asSetMap = M.fromList $ map (\(node, list) -> (node, S.fromList list)) postDoms + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) diff --git a/src/ShellCheck/CFGAnalysis.hs b/src/ShellCheck/CFGAnalysis.hs index bb90860..dc0a4b1 100644 --- a/src/ShellCheck/CFGAnalysis.hs +++ b/src/ShellCheck/CFGAnalysis.hs @@ -99,6 +99,7 @@ data CFGAnalysis = CFGAnalysis { graph :: CFGraph, tokenToRange :: M.Map Id (Node, Node), tokenToNodes :: M.Map Id (S.Set Node), + postDominators :: M.Map Node (S.Set Node), nodeToData :: M.Map Node (ProgramState, ProgramState) } deriving (Show, Generic, NFData) @@ -1304,7 +1305,8 @@ analyzeControlFlow params t = graph = cfGraph cfg, tokenToRange = cfIdToRange cfg, tokenToNodes = cfIdToNodes cfg, - nodeToData = nodeToData + nodeToData = nodeToData, + postDominators = cfPostDominators cfg } @@ -1355,5 +1357,6 @@ analyzeStragglers ctx state stragglers = do transferFunctionValue ctx def + return [] runTests = $quickCheckAll From e9784fa9a77f6f018997f5d164148422f7da0b33 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 25 Jul 2022 11:57:04 -0700 Subject: [PATCH 257/459] Refine #2544 to not warn when $? postdominates [ ] (fixes #2544) --- src/ShellCheck/Analytics.hs | 13 +++++++++++-- src/ShellCheck/CFGAnalysis.hs | 10 ++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 8499d8d..42d5a9e 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -4884,7 +4884,11 @@ checkCommandIsUnreachable params t = prop_checkOverwrittenExitCode1 = verify checkOverwrittenExitCode "x; [ $? -eq 1 ] || [ $? -eq 2 ]" prop_checkOverwrittenExitCode2 = verifyNot checkOverwrittenExitCode "x; [ $? -eq 1 ]" prop_checkOverwrittenExitCode3 = verify checkOverwrittenExitCode "x; echo \"Exit is $?\"; [ $? -eq 0 ]" -prop_checkOverwrittenExitCode4 = verifyNot checkOverwrittenExitCode "x; [ $? -eq 0 ]" +prop_checkOverwrittenExitCode4 = verifyNot checkOverwrittenExitCode "x; [ $? -eq 0 ] && echo Success" +prop_checkOverwrittenExitCode5 = verify checkOverwrittenExitCode "x; if [ $? -eq 0 ]; then var=$?; fi" +prop_checkOverwrittenExitCode6 = verify checkOverwrittenExitCode "x; [ $? -gt 0 ] && fail=$?" +prop_checkOverwrittenExitCode7 = verifyNot checkOverwrittenExitCode "[ 1 -eq 2 ]; status=$?" +prop_checkOverwrittenExitCode8 = verifyNot checkOverwrittenExitCode "[ 1 -eq 2 ]; exit $?" checkOverwrittenExitCode params t = case t of T_DollarBraced id _ val | getLiteralString val == Just "?" -> check id @@ -4898,7 +4902,7 @@ checkOverwrittenExitCode params t = let idToToken = idMap params exitCodeTokens <- sequence $ map (\k -> Map.lookup k idToToken) $ S.toList exitCodeIds return $ do - when (all isCondition exitCodeTokens) $ + when (all isCondition exitCodeTokens && not (usedUnconditionally t exitCodeIds)) $ warn id 2319 "This $? refers to a condition, not a command. Assign to a variable to avoid it being overwritten." when (all isPrinting exitCodeTokens) $ warn id 2320 "This $? refers to echo/printf, not a previous command. Assign to variable to avoid it being overwritten." @@ -4909,6 +4913,11 @@ checkOverwrittenExitCode params t = T_SimpleCommand {} -> getCommandName t == Just "test" _ -> False + -- If we don't do anything based on the condition, assume we wanted the condition itself + -- This helps differentiate `x; [ $? -gt 0 ] && exit $?` vs `[ cond ]; exit $?` + usedUnconditionally t testIds = + all (\c -> CF.doesPostDominate (cfgAnalysis params) (getId t) c) testIds + isPrinting t = case getCommandBasename t of Just "echo" -> True diff --git a/src/ShellCheck/CFGAnalysis.hs b/src/ShellCheck/CFGAnalysis.hs index dc0a4b1..ff88810 100644 --- a/src/ShellCheck/CFGAnalysis.hs +++ b/src/ShellCheck/CFGAnalysis.hs @@ -56,6 +56,7 @@ module ShellCheck.CFGAnalysis ( ,SpaceStatus (..) ,getIncomingState ,getOutgoingState + ,doesPostDominate ,ShellCheck.CFGAnalysis.runTests -- STRIP ) where @@ -140,6 +141,15 @@ getOutgoingState analysis id = do (start,end) <- M.lookup id $ tokenToRange analysis snd <$> M.lookup end (nodeToData analysis) +-- Conveniently determine whether one node postdominates another, +-- i.e. whether 'target' always unconditionally runs after 'base'. +doesPostDominate :: CFGAnalysis -> Id -> Id -> Bool +doesPostDominate analysis target base = fromMaybe False $ do + (_, baseEnd) <- M.lookup base $ tokenToRange analysis + (targetStart, _) <- M.lookup target $ tokenToRange analysis + postDoms <- M.lookup baseEnd $ postDominators analysis + return $ S.member targetStart postDoms + getDataForNode analysis node = M.lookup node $ nodeToData analysis -- The current state of data flow at a point in the program, potentially as a diff From c57e447c89a9ba64bd717560edbeb2192bb7b92a Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 26 Jul 2022 09:46:07 -0700 Subject: [PATCH 258/459] Correctly discard overlapping fixes in diff output (fixes #2370) --- src/ShellCheck/Formatter/Diff.hs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/ShellCheck/Formatter/Diff.hs b/src/ShellCheck/Formatter/Diff.hs index 197b3af..15d00d7 100644 --- a/src/ShellCheck/Formatter/Diff.hs +++ b/src/ShellCheck/Formatter/Diff.hs @@ -203,10 +203,9 @@ formatDoc color (DiffDoc name lf regions) = buildFixMap :: [Fix] -> M.Map String Fix buildFixMap fixes = perFile where - splitFixes = concatMap splitFixByFile fixes + splitFixes = splitFixByFile $ mconcat fixes perFile = groupByMap (posFile . repStartPos . head . fixReplacements) splitFixes --- There are currently no multi-file fixes, but let's handle it anyways splitFixByFile :: Fix -> [Fix] splitFixByFile fix = map makeFix $ groupBy sameFile (fixReplacements fix) where From b5f5e6347d59be6b26ce8761646fe95ac7b2f3c7 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 26 Jul 2022 10:42:01 -0700 Subject: [PATCH 259/459] Discard next rather than existing fixes when they overlap --- src/ShellCheck/Fixer.hs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ShellCheck/Fixer.hs b/src/ShellCheck/Fixer.hs index 2376842..43a97ab 100644 --- a/src/ShellCheck/Fixer.hs +++ b/src/ShellCheck/Fixer.hs @@ -87,6 +87,7 @@ instance Ranged Replacement where instance Monoid Fix where mempty = newFix mappend = (<>) + mconcat = foldl mappend mempty -- fold left to right since <> discards right on overlap instance Semigroup Fix where f1 <> f2 = From 4a27c9a8d50996438ff4853206be674e28069386 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 26 Jul 2022 15:33:25 -0700 Subject: [PATCH 260/459] Fix overlap check --- src/ShellCheck/Fixer.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Fixer.hs b/src/ShellCheck/Fixer.hs index 43a97ab..358dec9 100644 --- a/src/ShellCheck/Fixer.hs +++ b/src/ShellCheck/Fixer.hs @@ -36,7 +36,7 @@ class Ranged a where end :: a -> Position overlap :: a -> a -> Bool overlap x y = - (yStart >= xStart && yStart < xEnd) || (yStart < xStart && yEnd > xStart) + xEnd > yStart && yEnd > xStart where yStart = start y yEnd = end y From a30ac402eb39d3e60cd3a64abd48d00c85d49bab Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 27 Jul 2022 11:29:55 -0700 Subject: [PATCH 261/459] Don't use & for updates as result is unspecified This fixes `Prelude.foldl1: empty list []` when script has `( exit )` --- src/ShellCheck/CFG.hs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/CFG.hs b/src/ShellCheck/CFG.hs index ad05e93..39747cf 100644 --- a/src/ShellCheck/CFG.hs +++ b/src/ShellCheck/CFG.hs @@ -1203,6 +1203,9 @@ tokenToParts t = _ -> [maybe CFStringUnknown CFStringLiteral $ getLiteralString t] +-- Like & but well defined when the node already exists +safeUpdate ctx@(_,node,_,_) graph = ctx & (delNode node graph) + -- Change all subshell invocations to instead link directly to their contents. -- This is used for producing dominator trees. inlineSubshells :: CFGraph -> CFGraph @@ -1223,7 +1226,7 @@ inlineSubshells graph = relinkedGraph endToNexts = (endIncoming, endNode, endLabel, outgoing) (endIncoming, endNode, endLabel, _) = context graph end in - subshellToStart & (endToNexts & graph) + subshellToStart `safeUpdate` (endToNexts `safeUpdate` graph) findEntryNodes :: CFGraph -> [Node] findEntryNodes graph = ufold find [] graph @@ -1259,7 +1262,7 @@ findPostDominators mainexit graph = asSetMap inlined = inlineSubshells graph terminals = findTerminalNodes inlined (incoming, _, label, outgoing) = context graph mainexit - withExitEdges = (incoming ++ map (\c -> (CFEFlow, c)) terminals, mainexit, label, outgoing) & inlined + withExitEdges = (incoming ++ map (\c -> (CFEFlow, c)) terminals, mainexit, label, outgoing) `safeUpdate` inlined reversed = grev withExitEdges postDoms = dom reversed mainexit asSetMap = M.fromList $ map (\(node, list) -> (node, S.fromList list)) postDoms From 3ce310e939427216e461681e0a71c4883ff5bf03 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 27 Jul 2022 14:25:19 -0700 Subject: [PATCH 262/459] Plug space leaks when processing multiple files --- shellcheck.hs | 2 +- src/ShellCheck/Formatter/JSON.hs | 3 ++- src/ShellCheck/Formatter/JSON1.hs | 3 ++- src/ShellCheck/Formatter/TTY.hs | 3 ++- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/shellcheck.hs b/shellcheck.hs index bf70445..a525251 100644 --- a/shellcheck.hs +++ b/shellcheck.hs @@ -225,7 +225,7 @@ runFormatter sys format options files = do f :: Status -> FilePath -> IO Status f status file = do newStatus <- process file `catch` handler file - return $ status `mappend` newStatus + return $! status `mappend` newStatus handler :: FilePath -> IOException -> IO Status handler file e = reportFailure file (show e) reportFailure file str = do diff --git a/src/ShellCheck/Formatter/JSON.hs b/src/ShellCheck/Formatter/JSON.hs index 7c26421..6b38532 100644 --- a/src/ShellCheck/Formatter/JSON.hs +++ b/src/ShellCheck/Formatter/JSON.hs @@ -23,6 +23,7 @@ module ShellCheck.Formatter.JSON (format) where import ShellCheck.Interface import ShellCheck.Formatter.Format +import Control.DeepSeq import Data.Aeson import Data.IORef import Data.Monoid @@ -103,7 +104,7 @@ collectResult ref cr sys = mapM_ f groups comments = crComments cr groups = groupWith sourceFile comments f :: [PositionedComment] -> IO () - f group = modifyIORef ref (\x -> comments ++ x) + f group = deepseq comments $ modifyIORef ref (\x -> comments ++ x) finish ref = do list <- readIORef ref diff --git a/src/ShellCheck/Formatter/JSON1.hs b/src/ShellCheck/Formatter/JSON1.hs index 54aad34..2169bf6 100644 --- a/src/ShellCheck/Formatter/JSON1.hs +++ b/src/ShellCheck/Formatter/JSON1.hs @@ -23,6 +23,7 @@ module ShellCheck.Formatter.JSON1 (format) where import ShellCheck.Interface import ShellCheck.Formatter.Format +import Control.DeepSeq import Data.Aeson import Data.IORef import Data.Monoid @@ -120,7 +121,7 @@ collectResult ref cr sys = mapM_ f groups result <- siReadFile sys (Just True) filename let contents = either (const "") id result let comments' = makeNonVirtual comments contents - modifyIORef ref (\x -> comments' ++ x) + deepseq comments' $ modifyIORef ref (\x -> comments' ++ x) finish ref = do list <- readIORef ref diff --git a/src/ShellCheck/Formatter/TTY.hs b/src/ShellCheck/Formatter/TTY.hs index 8dd90d4..e28696c 100644 --- a/src/ShellCheck/Formatter/TTY.hs +++ b/src/ShellCheck/Formatter/TTY.hs @@ -23,6 +23,7 @@ import ShellCheck.Fixer import ShellCheck.Interface import ShellCheck.Formatter.Format +import Control.DeepSeq import Control.Monad import Data.Array import Data.Foldable @@ -88,7 +89,7 @@ rankError err = (ranking, cSeverity $ pcComment err, cCode $ pcComment err) appendComments errRef comments max = do previous <- readIORef errRef let current = map (\x -> (rankError x, cCode $ pcComment x, cMessage $ pcComment x)) comments - writeIORef errRef . take max . nubBy equal . sort $ previous ++ current + writeIORef errRef $! force . take max . nubBy equal . sort $ previous ++ current where fst3 (x,_,_) = x equal x y = fst3 x == fst3 y From f4409122799d7967ff843259bc3a51e2fcad2cef Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 27 Jul 2022 19:47:37 -0700 Subject: [PATCH 263/459] Refactor to not generate Parameters twice --- src/ShellCheck/ASTLib.hs | 4 ++++ src/ShellCheck/Analytics.hs | 39 +++++++++++++++---------------------- src/ShellCheck/Analyzer.hs | 4 ++-- src/ShellCheck/Checker.hs | 5 +++-- 4 files changed, 25 insertions(+), 27 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 7b4f9e5..56903ee 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -898,6 +898,10 @@ isClosingFileOp op = T_IoDuplicate _ (T_LESSAND _) "-" -> True _ -> False +getEnableDirectives root = + case root of + T_Annotation _ list _ -> [s | EnableComment s <- list] + _ -> [] return [] runTests = $quickCheckAll diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 42d5a9e..e878dc4 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -19,7 +19,7 @@ -} {-# LANGUAGE TemplateHaskell #-} {-# LANGUAGE FlexibleContexts #-} -module ShellCheck.Analytics (runAnalytics, optionalChecks, ShellCheck.Analytics.runTests) where +module ShellCheck.Analytics (checker, optionalChecks, ShellCheck.Analytics.runTests) where import ShellCheck.AST import ShellCheck.ASTLib @@ -71,29 +71,22 @@ treeChecks = [ ,checkArrayValueUsedAsIndex ] -runAnalytics :: AnalysisSpec -> [TokenComment] -runAnalytics options = - runList options treeChecks ++ runList options optionalChecks +checker spec params = mkChecker spec params treeChecks + +mkChecker spec params checks = + Checker { + perScript = \(Root root) -> do + tell $ concatMap (\f -> f params root) all, + perToken = const $ return () + } where - root = asScript options - optionals = getEnableDirectives root ++ asOptionalChecks options - optionalChecks = - if "all" `elem` optionals + all = checks ++ optionals + optionalKeys = asOptionalChecks spec + optionals = + if "all" `elem` optionalKeys then map snd optionalTreeChecks - else mapMaybe (\c -> Map.lookup c optionalCheckMap) optionals + else mapMaybe (\c -> Map.lookup c optionalCheckMap) optionalKeys -runList :: AnalysisSpec -> [Parameters -> Token -> [TokenComment]] - -> [TokenComment] -runList spec list = notes - where - root = asScript spec - params = makeParameters spec - notes = concatMap (\f -> f params root) list - -getEnableDirectives root = - case root of - T_Annotation _ list _ -> [s | EnableComment s <- list] - _ -> [] checkList l t = concatMap (\f -> f t) l @@ -318,12 +311,12 @@ producesComments f s = not . null <$> runAndGetComments f s runAndGetComments f s = do let pr = pScript s - prRoot pr + root <- prRoot pr let spec = defaultSpec pr let params = makeParameters spec return $ filterByAnnotation spec params $ - runList spec [f] + f params root -- Copied from https://wiki.haskell.org/Edit_distance dist :: Eq a => [a] -> [a] -> Int diff --git a/src/ShellCheck/Analyzer.hs b/src/ShellCheck/Analyzer.hs index ff2e457..06b6e53 100644 --- a/src/ShellCheck/Analyzer.hs +++ b/src/ShellCheck/Analyzer.hs @@ -35,13 +35,13 @@ analyzeScript :: AnalysisSpec -> AnalysisResult analyzeScript spec = newAnalysisResult { arComments = filterByAnnotation spec params . nub $ - runAnalytics spec - ++ runChecker params (checkers spec params) + runChecker params (checkers spec params) } where params = makeParameters spec checkers spec params = mconcat $ map ($ params) [ + ShellCheck.Analytics.checker spec, ShellCheck.Checks.Commands.checker spec, ShellCheck.Checks.ControlFlow.checker spec, ShellCheck.Checks.Custom.checker, diff --git a/src/ShellCheck/Checker.hs b/src/ShellCheck/Checker.hs index ef8182f..db793f1 100644 --- a/src/ShellCheck/Checker.hs +++ b/src/ShellCheck/Checker.hs @@ -20,9 +20,10 @@ {-# LANGUAGE TemplateHaskell #-} module ShellCheck.Checker (checkScript, ShellCheck.Checker.runTests) where +import ShellCheck.Analyzer +import ShellCheck.ASTLib import ShellCheck.Interface import ShellCheck.Parser -import ShellCheck.Analyzer import Data.Either import Data.Functor @@ -85,7 +86,7 @@ checkScript sys spec = do asCheckSourced = csCheckSourced spec, asExecutionMode = Executed, asTokenPositions = tokenPositions, - asOptionalChecks = csOptionalChecks spec + asOptionalChecks = getEnableDirectives root ++ csOptionalChecks spec } where as = newAnalysisSpec root let analysisMessages = maybe [] From d0dd81e1faa506232193ad91030dd2cb9f2d4a66 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 28 Jul 2022 08:56:44 -0700 Subject: [PATCH 264/459] Allow quoting values in directives (fixes #2517) --- CHANGELOG.md | 1 + shellcheck.1.md | 3 +++ src/ShellCheck/Checker.hs | 12 ++++++++++++ src/ShellCheck/Parser.hs | 23 +++++++++++++++++------ 4 files changed, 33 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cef16f4..c363eb5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ based on control flow rather than just syntax. Existing checks will gradually start using it, which may cause them to trigger differently (but more accurately). +- Values in directives/shellcheckrc can now be quoted with '' or "" ## v0.8.0 - 2021-11-06 diff --git a/shellcheck.1.md b/shellcheck.1.md index 146d791..c345a2b 100644 --- a/shellcheck.1.md +++ b/shellcheck.1.md @@ -282,6 +282,9 @@ Here is an example `.shellcheckrc`: source-path=SCRIPTDIR source-path=/mnt/chroot + # Since 0.9.0, values can be quoted with '' or "" to allow spaces + source-path="My Documents/scripts" + # Allow opening any 'source'd file, even if not specified as input external-sources=true diff --git a/src/ShellCheck/Checker.hs b/src/ShellCheck/Checker.hs index db793f1..6518e0d 100644 --- a/src/ShellCheck/Checker.hs +++ b/src/ShellCheck/Checker.hs @@ -244,6 +244,9 @@ prop_canStripPrefixAndSource2 = prop_canSourceDynamicWhenRedirected = null $ checkWithIncludes [("lib", "")] "#shellcheck source=lib\n. \"$1\"" +prop_canRedirectWithSpaces = + null $ checkWithIncludes [("my file", "")] "#shellcheck source=\"my file\"\n. \"$1\"" + prop_recursiveAnalysis = [2086] == checkRecursive [("lib", "echo $1")] "source lib" @@ -413,6 +416,15 @@ prop_sourcePathAddsAnnotation = result == [2086] csCheckSourced = True } +prop_sourcePathWorksWithSpaces = result == [2086] + where + f "dir/myscript" _ ["my path"] "lib" = return "foo/lib" + result = checkWithIncludesAndSourcePath [("foo/lib", "echo $1")] f emptyCheckSpec { + csScript = "#!/bin/bash\n# shellcheck source-path='my path'\nsource lib", + csFilename = "dir/myscript", + csCheckSourced = True + } + prop_sourcePathRedirectsDirective = result == [2086] where f "dir/myscript" _ _ "lib" = return "foo/lib" diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 0dd6621..4ff45ed 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -992,6 +992,10 @@ prop_readAnnotation5 = isOk readAnnotation "# shellcheck disable=SC2002 # All ca prop_readAnnotation6 = isOk readAnnotation "# shellcheck disable=SC1234 # shellcheck foo=bar\n" prop_readAnnotation7 = isOk readAnnotation "# shellcheck disable=SC1000,SC2000-SC3000,SC1001\n" prop_readAnnotation8 = isOk readAnnotation "# shellcheck disable=all\n" +prop_readAnnotation9 = isOk readAnnotation "# shellcheck source='foo bar' source-path=\"baz etc\"\n" +prop_readAnnotation10 = isOk readAnnotation "# shellcheck disable='SC1234,SC2345' enable=\"foo\" shell='bash'\n" +prop_readAnnotation11 = isOk (readAnnotationWithoutPrefix False) "external-sources='true'" + readAnnotation = called "shellcheck directive" $ do try readAnnotationPrefix many1 linewhitespace @@ -1007,12 +1011,19 @@ readAnnotationWithoutPrefix sandboxed = do many linewhitespace return $ concat values where + plainOrQuoted p = quoted p <|> p + quoted p = do + c <- oneOf "'\"" + start <- getPosition + str <- many1 $ noneOf (c:"\n") + char c <|> fail "Missing terminating quote for directive." + subParse start p str readKey = do keyPos <- getPosition key <- many1 (letter <|> char '-') char '=' <|> fail "Expected '=' after directive key" annotations <- case key of - "disable" -> readElement `sepBy` char ',' + "disable" -> plainOrQuoted $ readElement `sepBy` char ',' where readElement = readRange <|> readAll readAll = do @@ -1027,21 +1038,21 @@ readAnnotationWithoutPrefix sandboxed = do int <- many1 digit return $ read int - "enable" -> readName `sepBy` char ',' + "enable" -> plainOrQuoted $ readName `sepBy` char ',' where readName = EnableComment <$> many1 (letter <|> char '-') "source" -> do - filename <- many1 $ noneOf " \n" + filename <- quoted (many1 anyChar) <|> (many1 $ noneOf " \n") return [SourceOverride filename] "source-path" -> do - dirname <- many1 $ noneOf " \n" + dirname <- quoted (many1 anyChar) <|> (many1 $ noneOf " \n") return [SourcePath dirname] "shell" -> do pos <- getPosition - shell <- many1 $ noneOf " \n" + shell <- quoted (many1 anyChar) <|> (many1 $ noneOf " \n") when (isNothing $ shellForExecutable shell) $ parseNoteAt pos ErrorC 1103 "This shell type is unknown. Use e.g. sh or bash." @@ -1049,7 +1060,7 @@ readAnnotationWithoutPrefix sandboxed = do "external-sources" -> do pos <- getPosition - value <- many1 letter + value <- plainOrQuoted $ many1 letter case value of "true" -> if sandboxed From c76b8d9a327812fe8164e9b667650d0c183978d8 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 28 Jul 2022 09:37:23 -0700 Subject: [PATCH 265/459] Let annotations take effect earlier (fixes #2534) --- src/ShellCheck/Checker.hs | 11 ++++++ src/ShellCheck/Parser.hs | 80 +++++++++++++++++++-------------------- 2 files changed, 49 insertions(+), 42 deletions(-) diff --git a/src/ShellCheck/Checker.hs b/src/ShellCheck/Checker.hs index 6518e0d..c8d2c39 100644 --- a/src/ShellCheck/Checker.hs +++ b/src/ShellCheck/Checker.hs @@ -496,6 +496,17 @@ prop_fileCannotEnableExternalSources2 = result == [1144] csCheckSourced = True } +prop_rcCanSuppressEarlyProblems1 = null result + where + result = checkWithRc "disable=1071" emptyCheckSpec { + csScript = "#!/bin/zsh\necho $1" + } + +prop_rcCanSuppressEarlyProblems2 = null result + where + result = checkWithRc "disable=1104" emptyCheckSpec { + csScript = "!/bin/bash\necho 'hello world'" + } return [] runTests = $quickCheckAll diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 4ff45ed..d461fc7 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -38,7 +38,6 @@ import Data.Functor import Data.List (isPrefixOf, isInfixOf, isSuffixOf, partition, sortBy, intercalate, nub, find) import Data.Maybe import Data.Monoid -import Debug.Trace -- STRIP import GHC.Exts (sortWith) import Prelude hiding (readList) import System.IO @@ -458,8 +457,8 @@ called s p = do pos <- getPosition withContext (ContextName pos s) p -withAnnotations anns = - withContext (ContextAnnotation anns) +withAnnotations anns p = + if null anns then p else withContext (ContextAnnotation anns) p readConditionContents single = readCondContents `attempting` lookAhead (do @@ -3258,44 +3257,51 @@ prop_readScript3 = isWarning readScript "#!/bin/bash\necho hello\xA0world" prop_readScript4 = isWarning readScript "#!/usr/bin/perl\nfoo=(" prop_readScript5 = isOk readScript "#!/bin/bash\n#This is an empty script\n\n" prop_readScript6 = isOk readScript "#!/usr/bin/env -S X=FOO bash\n#This is an empty script\n\n" +prop_readScript7 = isOk readScript "#!/bin/zsh\n# shellcheck disable=SC1071\nfor f (a b); echo $f\n" readScriptFile sourced = do start <- startSpan pos <- getPosition - optional $ do - readUtf8Bom - parseProblem ErrorC 1082 - "This file has a UTF-8 BOM. Remove it with: LC_CTYPE=C sed '1s/^...//' < yourscript ." - shebang <- readShebang <|> readEmptyLiteral - let (T_Literal _ shebangString) = shebang - allspacing - annotationStart <- startSpan - fileAnnotations <- readAnnotations rcAnnotations <- if sourced then return [] else do filename <- Mr.asks currentFilename readConfigFile filename - let annotations = fileAnnotations ++ rcAnnotations - annotationId <- endSpan annotationStart - let shellAnnotationSpecified = - any (\x -> case x of ShellOverride {} -> True; _ -> False) annotations - shellFlagSpecified <- isJust <$> Mr.asks shellTypeOverride - let ignoreShebang = shellAnnotationSpecified || shellFlagSpecified - unless ignoreShebang $ - verifyShebang pos (executableFromShebang shebangString) - if ignoreShebang || isValidShell (executableFromShebang shebangString) /= Just False - then do - commands <- withAnnotations annotations readCompoundListOrEmpty - id <- endSpan start - verifyEof - let script = T_Annotation annotationId annotations $ - T_Script id shebang commands - reparseIndices script - else do - many anyChar - id <- endSpan start - return $ T_Script id shebang [] + -- Put the rc annotations on the stack so that one can ignore e.g. SC1084 in .shellcheckrc + withAnnotations rcAnnotations $ do + hasBom <- wasIncluded readUtf8Bom + shebang <- readShebang <|> readEmptyLiteral + let (T_Literal _ shebangString) = shebang + allspacing + annotationStart <- startSpan + fileAnnotations <- readAnnotations + + -- Similarly put the filewide annotations on the stack to allow earlier suppression + withAnnotations fileAnnotations $ do + when (hasBom) $ + parseProblemAt pos ErrorC 1082 + "This file has a UTF-8 BOM. Remove it with: LC_CTYPE=C sed '1s/^...//' < yourscript ." + let annotations = fileAnnotations ++ rcAnnotations + annotationId <- endSpan annotationStart + let shellAnnotationSpecified = + any (\x -> case x of ShellOverride {} -> True; _ -> False) annotations + shellFlagSpecified <- isJust <$> Mr.asks shellTypeOverride + let ignoreShebang = shellAnnotationSpecified || shellFlagSpecified + + unless ignoreShebang $ + verifyShebang pos (executableFromShebang shebangString) + if ignoreShebang || isValidShell (executableFromShebang shebangString) /= Just False + then do + commands <- readCompoundListOrEmpty + id <- endSpan start + verifyEof + let script = T_Annotation annotationId annotations $ + T_Script id shebang commands + reparseIndices script + else do + many anyChar + id <- endSpan start + return $ T_Script id shebang [] where verifyShebang pos s = do @@ -3388,16 +3394,6 @@ parsesCleanly parser string = runIdentity $ do return $ Just . null $ parseNotes userState ++ parseProblems systemState (Left _, _) -> return Nothing --- For printf debugging: print the value of an expression --- Example: return $ dump $ T_Literal id [c] -dump :: Show a => a -> a -- STRIP -dump x = trace (show x) x -- STRIP - --- Like above, but print a specific expression: --- Example: return $ dumps ("Returning: " ++ [c]) $ T_Literal id [c] -dumps :: Show x => x -> a -> a -- STRIP -dumps t = trace (show t) -- STRIP - parseWithNotes parser = do item <- parser state <- getState From 04db46381fe515d6e3e66b7d3c33c60fa0275471 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 28 Jul 2022 19:00:03 -0700 Subject: [PATCH 266/459] Use Data.Map.Strict instead for a ~15% parsing speedup --- src/ShellCheck/Parser.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index d461fc7..aeaf703 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -46,7 +46,7 @@ import Text.Parsec.Error import Text.Parsec.Pos import qualified Control.Monad.Reader as Mr import qualified Control.Monad.State as Ms -import qualified Data.Map as Map +import qualified Data.Map.Strict as Map import Test.QuickCheck.All (quickCheckAll) From 77069f7445681747ff88eaa56b6bd23b596eee99 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Fri, 29 Jul 2022 21:05:33 -0700 Subject: [PATCH 267/459] Store postdominators as Array Node [Node] for a significant win --- src/ShellCheck/CFG.hs | 17 ++++++++++------- src/ShellCheck/CFGAnalysis.hs | 8 ++++---- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/ShellCheck/CFG.hs b/src/ShellCheck/CFG.hs index 39747cf..771e870 100644 --- a/src/ShellCheck/CFG.hs +++ b/src/ShellCheck/CFG.hs @@ -47,6 +47,8 @@ import ShellCheck.Regex import Control.DeepSeq import Control.Monad import Control.Monad.Identity +import Data.Array.Unboxed +import Data.Array.ST import Data.List hiding (map) import Data.Maybe import qualified Data.Map as M @@ -174,10 +176,10 @@ data CFGResult = CFGResult { cfIdToRange :: M.Map Id (Node, Node), -- A set of all nodes belonging to an Id, recursively cfIdToNodes :: M.Map Id (S.Set Node), - -- A map to nodes that the given node postdominates - cfPostDominators :: M.Map Node (S.Set Node) + -- An array (from,to) saying whether 'from' postdominates 'to' + cfPostDominators :: Array Node [Node] } - deriving (Show, Generic, NFData) + deriving (Show) buildGraph :: CFGParameters -> Token -> CFGResult buildGraph params root = @@ -200,7 +202,7 @@ buildGraph params root = cfPostDominators = findPostDominators mainExit $ mkGraph nodes onlyRealEdges } in - deepseq result result + result remapGraph :: M.Map Node Node -> CFW -> CFW remapGraph remap (nodes, edges, mapping, assoc) = @@ -1256,8 +1258,8 @@ findTerminalNodes graph = ufold find [] graph f (IdTagged _ (CFDefineFunction _ id start end):rest) list = f rest (end:list) f (_:rest) list = f rest list -findPostDominators :: Node -> CFGraph -> M.Map Node (S.Set Node) -findPostDominators mainexit graph = asSetMap +findPostDominators :: Node -> CFGraph -> Array Node [Node] +findPostDominators mainexit graph = asArray where inlined = inlineSubshells graph terminals = findTerminalNodes inlined @@ -1265,7 +1267,8 @@ findPostDominators mainexit graph = asSetMap withExitEdges = (incoming ++ map (\c -> (CFEFlow, c)) terminals, mainexit, label, outgoing) `safeUpdate` inlined reversed = grev withExitEdges postDoms = dom reversed mainexit - asSetMap = M.fromList $ map (\(node, list) -> (node, S.fromList list)) postDoms + (_, maxNode) = nodeRange graph + asArray = array (0, maxNode) postDoms return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) diff --git a/src/ShellCheck/CFGAnalysis.hs b/src/ShellCheck/CFGAnalysis.hs index ff88810..e6b1701 100644 --- a/src/ShellCheck/CFGAnalysis.hs +++ b/src/ShellCheck/CFGAnalysis.hs @@ -69,6 +69,7 @@ import Control.Monad import Control.Monad.ST import Control.DeepSeq import Data.List hiding (map) +import Data.Array.Unboxed import Data.STRef import Data.Maybe import qualified Data.Map as M @@ -100,9 +101,9 @@ data CFGAnalysis = CFGAnalysis { graph :: CFGraph, tokenToRange :: M.Map Id (Node, Node), tokenToNodes :: M.Map Id (S.Set Node), - postDominators :: M.Map Node (S.Set Node), + postDominators :: Array Node [Node], nodeToData :: M.Map Node (ProgramState, ProgramState) -} deriving (Show, Generic, NFData) +} deriving (Show) -- The program state we expose externally data ProgramState = ProgramState { @@ -147,8 +148,7 @@ doesPostDominate :: CFGAnalysis -> Id -> Id -> Bool doesPostDominate analysis target base = fromMaybe False $ do (_, baseEnd) <- M.lookup base $ tokenToRange analysis (targetStart, _) <- M.lookup target $ tokenToRange analysis - postDoms <- M.lookup baseEnd $ postDominators analysis - return $ S.member targetStart postDoms + return $ targetStart `elem` (postDominators analysis ! baseEnd) getDataForNode analysis node = M.lookup node $ nodeToData analysis From 0df934514298adadc40651696d5b854784efa0a5 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 2 Aug 2022 11:25:35 -0700 Subject: [PATCH 268/459] Trace numerical status, use for SC2071 (ref #2541) --- src/ShellCheck/Analytics.hs | 21 +++++++-- src/ShellCheck/CFGAnalysis.hs | 84 ++++++++++++++++++++++++++--------- 2 files changed, 82 insertions(+), 23 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index e878dc4..b5bac35 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1167,6 +1167,10 @@ prop_checkNumberComparisons18 = verify checkNumberComparisons "[[ foo -eq 2 ]]" prop_checkNumberComparisons19 = verifyNot checkNumberComparisons "foo=1; [[ foo -eq 2 ]]" prop_checkNumberComparisons20 = verify checkNumberComparisons "[[ 2 -eq / ]]" prop_checkNumberComparisons21 = verify checkNumberComparisons "[[ foo -eq foo ]]" +prop_checkNumberComparisons22 = verify checkNumberComparisons "x=10; [[ $x > $z ]]" +prop_checkNumberComparisons23 = verify checkNumberComparisons "x=0; if [[ -n $def ]]; then x=$def; fi; while [ $x > $z ]; do lol; done" +prop_checkNumberComparisons24 = verify checkNumberComparisons "x=$RANDOM; [ $x > $z ]" +prop_checkNumberComparisons25 = verify checkNumberComparisons "[[ $((n++)) > $x ]]" checkNumberComparisons params (TC_Binary id typ op lhs rhs) = do if isNum lhs || isNum rhs @@ -1242,9 +1246,20 @@ checkNumberComparisons params (TC_Binary id typ op lhs rhs) = do numChar x = isDigit x || x `elem` "+-. " isNum t = - case oversimplify t of - [v] -> all isDigit v - _ -> False + case getWordParts t of + [T_DollarArithmetic {}] -> True + [b@(T_DollarBraced id _ c)] -> + let + str = concat $ oversimplify c + var = getBracedReference str + in fromMaybe False $ do + state <- CF.getIncomingState (cfgAnalysis params) id + value <- Map.lookup var $ CF.variablesInScope state + return $ CF.numericalStatus (CF.variableValue value) >= CF.NumericalStatusMaybe + _ -> + case oversimplify t of + [v] -> all isDigit v + _ -> False isFraction t = case oversimplify t of diff --git a/src/ShellCheck/CFGAnalysis.hs b/src/ShellCheck/CFGAnalysis.hs index e6b1701..634d354 100644 --- a/src/ShellCheck/CFGAnalysis.hs +++ b/src/ShellCheck/CFGAnalysis.hs @@ -54,29 +54,31 @@ module ShellCheck.CFGAnalysis ( ,VariableValue (..) ,VariableProperties ,SpaceStatus (..) + ,NumericalStatus (..) ,getIncomingState ,getOutgoingState ,doesPostDominate ,ShellCheck.CFGAnalysis.runTests -- STRIP ) where -import GHC.Generics (Generic) -import ShellCheck.AST -import ShellCheck.CFG -import qualified ShellCheck.Data as Data -import ShellCheck.Prelude +import Control.DeepSeq import Control.Monad import Control.Monad.ST -import Control.DeepSeq -import Data.List hiding (map) import Data.Array.Unboxed -import Data.STRef -import Data.Maybe -import qualified Data.Map as M -import qualified Data.Set as S +import Data.Char import Data.Graph.Inductive.Graph import Data.Graph.Inductive.Query.DFS +import Data.List hiding (map) +import Data.Maybe +import Data.STRef import Debug.Trace -- STRIP +import GHC.Generics (Generic) +import qualified Data.Map as M +import qualified Data.Set as S +import qualified ShellCheck.Data as Data +import ShellCheck.AST +import ShellCheck.CFG +import ShellCheck.Prelude import Test.QuickCheck @@ -183,16 +185,20 @@ createEnvironmentState = do foldl' (flip ($)) newInternalState $ concat [ addVars Data.internalVariables unknownVariableState, addVars Data.variablesWithoutSpaces spacelessVariableState, - addVars Data.specialIntegerVariables spacelessVariableState + addVars Data.specialIntegerVariables integerVariableState ] where addVars names val = map (\name -> insertGlobal name val) names spacelessVariableState = unknownVariableState { variableValue = VariableValue { literalValue = Nothing, - spaceStatus = SpaceStatusClean + spaceStatus = SpaceStatusClean, + numericalStatus = NumericalStatusUnknown } } + integerVariableState = unknownVariableState { + variableValue = unknownIntegerValue + } modified s = s { sVersion = -1 } @@ -289,7 +295,8 @@ unknownFunctionValue = S.singleton FunctionUnknown -- The information about the value of a single variable data VariableValue = VariableValue { literalValue :: Maybe String, -- TODO: For debugging. Remove me. - spaceStatus :: SpaceStatus + spaceStatus :: SpaceStatus, + numericalStatus :: NumericalStatus } deriving (Show, Eq, Ord, Generic, NFData) @@ -301,6 +308,9 @@ data VariableState = VariableState { -- Whether or not the value needs quoting (has spaces/globs), or we don't know data SpaceStatus = SpaceStatusEmpty | SpaceStatusClean | SpaceStatusDirty deriving (Show, Eq, Ord, Generic, NFData) +-- +-- Whether or not the value needs quoting (has spaces/globs), or we don't know +data NumericalStatus = NumericalStatusUnknown | NumericalStatusEmpty | NumericalStatusMaybe | NumericalStatusDefinitely deriving (Show, Eq, Ord, Generic, NFData) -- The set of possible sets of properties for this variable type VariableProperties = S.Set (S.Set CFVariableProp) @@ -314,12 +324,14 @@ unknownVariableState = VariableState { unknownVariableValue = VariableValue { literalValue = Nothing, - spaceStatus = SpaceStatusDirty + spaceStatus = SpaceStatusDirty, + numericalStatus = NumericalStatusUnknown } emptyVariableValue = unknownVariableValue { literalValue = Just "", - spaceStatus = SpaceStatusEmpty + spaceStatus = SpaceStatusEmpty, + numericalStatus = NumericalStatusEmpty } unsetVariableState = VariableState { @@ -334,7 +346,8 @@ mergeVariableState a b = VariableState { mergeVariableValue a b = VariableValue { literalValue = if literalValue a == literalValue b then literalValue a else Nothing, - spaceStatus = mergeSpaceStatus (spaceStatus a) (spaceStatus b) + spaceStatus = mergeSpaceStatus (spaceStatus a) (spaceStatus b), + numericalStatus = mergeNumericalStatus (numericalStatus a) (numericalStatus b) } mergeSpaceStatus a b = @@ -344,6 +357,16 @@ mergeSpaceStatus a b = (SpaceStatusClean, SpaceStatusClean) -> SpaceStatusClean _ -> SpaceStatusDirty +mergeNumericalStatus a b = + case (a,b) of + (NumericalStatusDefinitely, NumericalStatusDefinitely) -> NumericalStatusDefinitely + (NumericalStatusDefinitely, _) -> NumericalStatusMaybe + (_, NumericalStatusDefinitely) -> NumericalStatusMaybe + (NumericalStatusMaybe, _) -> NumericalStatusMaybe + (_, NumericalStatusMaybe) -> NumericalStatusMaybe + (NumericalStatusEmpty, NumericalStatusEmpty) -> NumericalStatusEmpty + _ -> NumericalStatusUnknown + -- A VersionedMap is a Map that keeps an additional integer version to quickly determine if it has changed. -- * Version -1 means it's unknown (possibly and presumably changed) -- * Version 0 means it's empty @@ -1154,7 +1177,8 @@ appendVariableValue :: VariableValue -> VariableValue -> VariableValue appendVariableValue a b = unknownVariableValue { literalValue = liftM2 (++) (literalValue a) (literalValue b), - spaceStatus = appendSpaceStatus (spaceStatus a) (spaceStatus b) + spaceStatus = appendSpaceStatus (spaceStatus a) (spaceStatus b), + numericalStatus = appendNumericalStatus (numericalStatus a) (numericalStatus b) } appendSpaceStatus a b = @@ -1164,14 +1188,25 @@ appendSpaceStatus a b = (SpaceStatusClean, SpaceStatusClean) -> a _ ->SpaceStatusDirty +appendNumericalStatus a b = + case (a,b) of + (NumericalStatusEmpty, x) -> x + (x, NumericalStatusEmpty) -> x + (NumericalStatusDefinitely, NumericalStatusDefinitely) -> NumericalStatusDefinitely + (NumericalStatusUnknown, _) -> NumericalStatusUnknown + (_, NumericalStatusUnknown) -> NumericalStatusUnknown + _ -> NumericalStatusMaybe + unknownIntegerValue = unknownVariableValue { literalValue = Nothing, - spaceStatus = SpaceStatusClean + spaceStatus = SpaceStatusClean, + numericalStatus = NumericalStatusDefinitely } literalToVariableValue str = unknownVariableValue { literalValue = Just str, - spaceStatus = literalToSpaceStatus str + spaceStatus = literalToSpaceStatus str, + numericalStatus = literalToNumericalStatus str } withoutChanges ctx f = do @@ -1191,6 +1226,15 @@ literalToSpaceStatus str = _ | all (`notElem` " \t\n*?[") str -> SpaceStatusClean _ -> SpaceStatusDirty +-- Get the NumericalStatus for a literal string, i.e. whether it's an integer +literalToNumericalStatus str = + case str of + "" -> NumericalStatusEmpty + '-':rest -> if isNumeric rest then NumericalStatusDefinitely else NumericalStatusUnknown + rest -> if isNumeric rest then NumericalStatusDefinitely else NumericalStatusUnknown + where + isNumeric = all isDigit + type StateMap = M.Map Node (InternalState, InternalState) -- Classic, iterative Data Flow Analysis. See Wikipedia for a description of the process. From 4806719035de364dd2d49f97112dd8b3a255e3b0 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 2 Aug 2022 15:47:59 -0700 Subject: [PATCH 269/459] Handle variable assignments from `read` in CFG --- src/ShellCheck/CFG.hs | 35 ++++++++++++++++++++++++++++++++++- src/ShellCheck/Data.hs | 1 + 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/CFG.hs b/src/ShellCheck/CFG.hs index 771e870..e0c6267 100644 --- a/src/ShellCheck/CFG.hs +++ b/src/ShellCheck/CFG.hs @@ -41,6 +41,7 @@ module ShellCheck.CFG ( import GHC.Generics (Generic) import ShellCheck.AST import ShellCheck.ASTLib +import ShellCheck.Data import ShellCheck.Interface import ShellCheck.Prelude import ShellCheck.Regex @@ -936,6 +937,8 @@ handleCommand cmd vars args literalCmd = do Just "mapfile" -> regularExpansionWithStatus vars args $ handleMapfile args Just "readarray" -> regularExpansionWithStatus vars args $ handleMapfile args + Just "read" -> regularExpansionWithStatus vars args $ handleRead args + Just "DEFINE_boolean" -> regularExpansionWithStatus vars args $ handleDEFINE args Just "DEFINE_float" -> regularExpansionWithStatus vars args $ handleDEFINE args Just "DEFINE_integer" -> regularExpansionWithStatus vars args $ handleDEFINE args @@ -1113,7 +1116,7 @@ handleCommand cmd vars args literalCmd = do in IdTagged id $ CFWriteVariable name CFValueArray getFromArg = do - flags <- getGnuOpts "d:n:O:s:u:C:c:t" args + flags <- getGnuOpts flagsForMapfile args (_, arg) <- lookup "" flags name <- getLiteralString arg return (getId arg, name) @@ -1125,6 +1128,36 @@ handleCommand cmd vars args literalCmd = do guard $ isVariableName name return (getId c, name) + handleRead (cmd:args) = newNodeRange $ CFApplyEffects main + where + main = fromMaybe fallback $ do + flags <- getGnuOpts flagsForRead args + return $ fromMaybe (withFields flags) $ withArray flags + + withArray :: [(String, (Token, Token))] -> Maybe [IdTagged CFEffect] + withArray flags = do + (_, token) <- lookup "a" flags + return $ fromMaybe [] $ do + name <- getLiteralString token + return [ IdTagged (getId token) $ CFWriteVariable name CFValueArray ] + + withFields flags = mapMaybe getAssignment flags + + getAssignment :: (String, (Token, Token)) -> Maybe (IdTagged CFEffect) + getAssignment f = do + ("", (t, _)) <- return f + name <- getLiteralString t + return $ IdTagged (getId t) $ CFWriteVariable name CFValueString + + fallback = + let + names = reverse $ map fromJust $ takeWhile isJust $ map (\c -> sequence (getId c, getLiteralString c)) $ reverse args + namesOrDefault = if null names then [(getId cmd, "REPLY")] else names + hasDashA = any (== "a") $ map fst $ getGenericOpts args + value = if hasDashA then CFValueArray else CFValueString + in + map (\(id, name) -> IdTagged id $ CFWriteVariable name value) namesOrDefault + handleDEFINE (cmd:args) = newNodeRange $ CFApplyEffects $ maybeToList findVar where diff --git a/src/ShellCheck/Data.hs b/src/ShellCheck/Data.hs index fb82ca8..4090922 100644 --- a/src/ShellCheck/Data.hs +++ b/src/ShellCheck/Data.hs @@ -159,5 +159,6 @@ shellForExecutable name = _ -> Nothing flagsForRead = "sreu:n:N:i:p:a:t:" +flagsForMapfile = "d:n:O:s:u:C:c:t" declaringCommands = ["local", "declare", "export", "readonly", "typeset", "let"] From ccab132b385a215a37ce7e6f4d2601454b0437b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lawrence=20Vel=C3=A1zquez?= Date: Tue, 20 Sep 2022 17:36:46 -0400 Subject: [PATCH 270/459] Reflow lists of internal shell variables No functional changes; this just makes the next few commits cleaner. --- src/ShellCheck/Data.hs | 41 +++++++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/src/ShellCheck/Data.hs b/src/ShellCheck/Data.hs index 4090922..3c9013f 100644 --- a/src/ShellCheck/Data.hs +++ b/src/ShellCheck/Data.hs @@ -30,23 +30,26 @@ internalVariables = [ -- Bash "BASH", "BASHOPTS", "BASHPID", "BASH_ALIASES", "BASH_ARGC", - "BASH_ARGV", "BASH_CMDS", "BASH_COMMAND", "BASH_EXECUTION_STRING", - "BASH_LINENO", "BASH_REMATCH", "BASH_SOURCE", "BASH_SUBSHELL", - "BASH_VERSINFO", "BASH_VERSION", "COMP_CWORD", "COMP_KEY", - "COMP_LINE", "COMP_POINT", "COMP_TYPE", "COMP_WORDBREAKS", - "COMP_WORDS", "COPROC", "DIRSTACK", "EUID", "FUNCNAME", "GROUPS", - "HISTCMD", "HOSTNAME", "HOSTTYPE", "LINENO", "MACHTYPE", "MAPFILE", - "OLDPWD", "OPTARG", "OPTIND", "OSTYPE", "PIPESTATUS", "PPID", "PWD", - "RANDOM", "READLINE_LINE", "READLINE_POINT", "REPLY", "SECONDS", - "SHELLOPTS", "SHLVL", "UID", "BASH_ENV", "BASH_XTRACEFD", "CDPATH", - "COLUMNS", "COMPREPLY", "EMACS", "ENV", "FCEDIT", "FIGNORE", + "BASH_ARGV", "BASH_CMDS", "BASH_COMMAND", + "BASH_EXECUTION_STRING", "BASH_LINENO", + "BASH_REMATCH", "BASH_SOURCE", "BASH_SUBSHELL", "BASH_VERSINFO", + "BASH_VERSION", "COMP_CWORD", "COMP_KEY", "COMP_LINE", "COMP_POINT", + "COMP_TYPE", "COMP_WORDBREAKS", "COMP_WORDS", "COPROC", "DIRSTACK", + "EUID", "FUNCNAME", "GROUPS", "HISTCMD", + "HOSTNAME", "HOSTTYPE", "LINENO", "MACHTYPE", "MAPFILE", "OLDPWD", + "OPTARG", "OPTIND", "OSTYPE", "PIPESTATUS", "PPID", "PWD", "RANDOM", + "READLINE_LINE", "READLINE_POINT", + "REPLY", "SECONDS", "SHELLOPTS", "SHLVL", "UID", + "BASH_ENV", "BASH_XTRACEFD", "CDPATH", "COLUMNS", + "COMPREPLY", "EMACS", "ENV", "FCEDIT", "FIGNORE", "FUNCNEST", "GLOBIGNORE", "HISTCONTROL", "HISTFILE", "HISTFILESIZE", "HISTIGNORE", "HISTSIZE", "HISTTIMEFORMAT", "HOME", "HOSTFILE", "IFS", - "IGNOREEOF", "INPUTRC", "LANG", "LC_ALL", "LC_COLLATE", "LC_CTYPE", - "LC_MESSAGES", "LC_MONETARY", "LC_NUMERIC", "LC_TIME", "LINES", "MAIL", - "MAILCHECK", "MAILPATH", "OPTERR", "PATH", "POSIXLY_CORRECT", - "PROMPT_COMMAND", "PROMPT_DIRTRIM", "PS1", "PS2", "PS3", "PS4", "SHELL", - "TIMEFORMAT", "TMOUT", "TMPDIR", "auto_resume", "histchars", "COPROC", + "IGNOREEOF", "INPUTRC", "LANG", "LC_ALL", "LC_COLLATE", + "LC_CTYPE", "LC_MESSAGES", "LC_MONETARY", "LC_NUMERIC", "LC_TIME", + "LINES", "MAIL", "MAILCHECK", "MAILPATH", "OPTERR", "PATH", + "POSIXLY_CORRECT", "PROMPT_COMMAND", "PROMPT_DIRTRIM", "PS1", + "PS2", "PS3", "PS4", "SHELL", "TIMEFORMAT", "TMOUT", "TMPDIR", + "auto_resume", "histchars", "COPROC", -- Other "USER", "TZ", "TERM", "LOGNAME", "LD_LIBRARY_PATH", "LANGUAGE", "DISPLAY", @@ -68,9 +71,11 @@ specialIntegerVariables = [ specialVariablesWithoutSpaces = "-" : specialIntegerVariables variablesWithoutSpaces = specialVariablesWithoutSpaces ++ [ - "BASHPID", "BASH_ARGC", "BASH_LINENO", "BASH_SUBSHELL", "EUID", "LINENO", - "OPTIND", "PPID", "RANDOM", "SECONDS", "SHELLOPTS", "SHLVL", "UID", - "COLUMNS", "HISTFILESIZE", "HISTSIZE", "LINES" + "BASHPID", "BASH_ARGC", "BASH_LINENO", "BASH_SUBSHELL", "EUID", + "LINENO", "OPTIND", "PPID", "RANDOM", + "SECONDS", + "SHELLOPTS", "SHLVL", "UID", "COLUMNS", "HISTFILESIZE", + "HISTSIZE", "LINES" -- shflags , "FLAGS_ERROR", "FLAGS_FALSE", "FLAGS_TRUE" From f28462b01ca634b4649e0a6dd11a0a8a207d5729 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lawrence=20Vel=C3=A1zquez?= Date: Tue, 20 Sep 2022 19:10:39 -0400 Subject: [PATCH 271/459] Remove duplicate "COPROC" from internal vars list --- src/ShellCheck/Data.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Data.hs b/src/ShellCheck/Data.hs index 3c9013f..8c65474 100644 --- a/src/ShellCheck/Data.hs +++ b/src/ShellCheck/Data.hs @@ -49,7 +49,7 @@ internalVariables = [ "LINES", "MAIL", "MAILCHECK", "MAILPATH", "OPTERR", "PATH", "POSIXLY_CORRECT", "PROMPT_COMMAND", "PROMPT_DIRTRIM", "PS1", "PS2", "PS3", "PS4", "SHELL", "TIMEFORMAT", "TMOUT", "TMPDIR", - "auto_resume", "histchars", "COPROC", + "auto_resume", "histchars", -- Other "USER", "TZ", "TERM", "LOGNAME", "LD_LIBRARY_PATH", "LANGUAGE", "DISPLAY", From 966fb3e3dd3b49e64f022478f18f7aa4bad53a27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lawrence=20Vel=C3=A1zquez?= Date: Tue, 20 Sep 2022 19:12:05 -0400 Subject: [PATCH 272/459] Recognize more Bash internal variables - BASH_ARGV0, introduced in Bash 5.0 - BASH_COMPAT, 4.3 - BASH_LOADABLES_PATH, 4.4 - CHILD_MAX, 4.3 - EPOCHREALTIME, 5.0 - EPOCHSECONDS, 5.0 - EXECIGNORE, 4.4 - INSIDE_EMACS, 4.4 - PS0, 4.4 - READLINE_ARGUMENT, 5.2 - READLINE_MARK, 5.1 - SRANDOM, 5.1 Fixes #1780 and #2554. --- src/ShellCheck/Data.hs | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/ShellCheck/Data.hs b/src/ShellCheck/Data.hs index 8c65474..35f314f 100644 --- a/src/ShellCheck/Data.hs +++ b/src/ShellCheck/Data.hs @@ -30,24 +30,24 @@ internalVariables = [ -- Bash "BASH", "BASHOPTS", "BASHPID", "BASH_ALIASES", "BASH_ARGC", - "BASH_ARGV", "BASH_CMDS", "BASH_COMMAND", - "BASH_EXECUTION_STRING", "BASH_LINENO", + "BASH_ARGV", "BASH_ARGV0", "BASH_CMDS", "BASH_COMMAND", + "BASH_EXECUTION_STRING", "BASH_LINENO", "BASH_LOADABLES_PATH", "BASH_REMATCH", "BASH_SOURCE", "BASH_SUBSHELL", "BASH_VERSINFO", "BASH_VERSION", "COMP_CWORD", "COMP_KEY", "COMP_LINE", "COMP_POINT", "COMP_TYPE", "COMP_WORDBREAKS", "COMP_WORDS", "COPROC", "DIRSTACK", - "EUID", "FUNCNAME", "GROUPS", "HISTCMD", + "EPOCHREALTIME", "EPOCHSECONDS", "EUID", "FUNCNAME", "GROUPS", "HISTCMD", "HOSTNAME", "HOSTTYPE", "LINENO", "MACHTYPE", "MAPFILE", "OLDPWD", "OPTARG", "OPTIND", "OSTYPE", "PIPESTATUS", "PPID", "PWD", "RANDOM", - "READLINE_LINE", "READLINE_POINT", - "REPLY", "SECONDS", "SHELLOPTS", "SHLVL", "UID", - "BASH_ENV", "BASH_XTRACEFD", "CDPATH", "COLUMNS", - "COMPREPLY", "EMACS", "ENV", "FCEDIT", "FIGNORE", + "READLINE_ARGUMENT", "READLINE_LINE", "READLINE_MARK", "READLINE_POINT", + "REPLY", "SECONDS", "SHELLOPTS", "SHLVL", "SRANDOM", "UID", "BASH_COMPAT", + "BASH_ENV", "BASH_XTRACEFD", "CDPATH", "CHILD_MAX", "COLUMNS", + "COMPREPLY", "EMACS", "ENV", "EXECIGNORE", "FCEDIT", "FIGNORE", "FUNCNEST", "GLOBIGNORE", "HISTCONTROL", "HISTFILE", "HISTFILESIZE", "HISTIGNORE", "HISTSIZE", "HISTTIMEFORMAT", "HOME", "HOSTFILE", "IFS", - "IGNOREEOF", "INPUTRC", "LANG", "LC_ALL", "LC_COLLATE", + "IGNOREEOF", "INPUTRC", "INSIDE_EMACS", "LANG", "LC_ALL", "LC_COLLATE", "LC_CTYPE", "LC_MESSAGES", "LC_MONETARY", "LC_NUMERIC", "LC_TIME", "LINES", "MAIL", "MAILCHECK", "MAILPATH", "OPTERR", "PATH", - "POSIXLY_CORRECT", "PROMPT_COMMAND", "PROMPT_DIRTRIM", "PS1", + "POSIXLY_CORRECT", "PROMPT_COMMAND", "PROMPT_DIRTRIM", "PS0", "PS1", "PS2", "PS3", "PS4", "SHELL", "TIMEFORMAT", "TMOUT", "TMPDIR", "auto_resume", "histchars", @@ -72,9 +72,9 @@ specialVariablesWithoutSpaces = "-" : specialIntegerVariables variablesWithoutSpaces = specialVariablesWithoutSpaces ++ [ "BASHPID", "BASH_ARGC", "BASH_LINENO", "BASH_SUBSHELL", "EUID", - "LINENO", "OPTIND", "PPID", "RANDOM", - "SECONDS", - "SHELLOPTS", "SHLVL", "UID", "COLUMNS", "HISTFILESIZE", + "EPOCHREALTIME", "EPOCHSECONDS", "LINENO", "OPTIND", "PPID", "RANDOM", + "READLINE_ARGUMENT", "READLINE_MARK", "SECONDS", + "SHELLOPTS", "SHLVL", "SRANDOM", "UID", "COLUMNS", "HISTFILESIZE", "HISTSIZE", "LINES" -- shflags From 0845b8118352c929d414231af23d302cb748178d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lawrence=20Vel=C3=A1zquez?= Date: Tue, 20 Sep 2022 20:00:23 -0400 Subject: [PATCH 273/459] Add READLINE_POINT to list of variables without spaces --- src/ShellCheck/Data.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Data.hs b/src/ShellCheck/Data.hs index 35f314f..7d6f5b4 100644 --- a/src/ShellCheck/Data.hs +++ b/src/ShellCheck/Data.hs @@ -73,7 +73,7 @@ specialVariablesWithoutSpaces = "-" : specialIntegerVariables variablesWithoutSpaces = specialVariablesWithoutSpaces ++ [ "BASHPID", "BASH_ARGC", "BASH_LINENO", "BASH_SUBSHELL", "EUID", "EPOCHREALTIME", "EPOCHSECONDS", "LINENO", "OPTIND", "PPID", "RANDOM", - "READLINE_ARGUMENT", "READLINE_MARK", "SECONDS", + "READLINE_ARGUMENT", "READLINE_MARK", "READLINE_POINT", "SECONDS", "SHELLOPTS", "SHLVL", "SRANDOM", "UID", "COLUMNS", "HISTFILESIZE", "HISTSIZE", "LINES" From fcc473e27fffc2dfe404c24b30f2cbb7e897ace8 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 21 Sep 2022 18:11:18 -0700 Subject: [PATCH 274/459] Include inherited env for DFA of leftover functions (fixes #2560) --- src/ShellCheck/Analytics.hs | 2 ++ src/ShellCheck/CFGAnalysis.hs | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index b5bac35..e9ea36a 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2099,6 +2099,8 @@ prop_checkSpacefulnessCfg61 = verify checkSpacefulnessCfg "declare -x X; y=foo$X prop_checkSpacefulnessCfg62 = verifyNot checkSpacefulnessCfg "f() { declare -x X; y=foo$X; echo $y; }" prop_checkSpacefulnessCfg63 = verify checkSpacefulnessCfg "f && declare -i s; s='x + y'; echo $s" prop_checkSpacefulnessCfg64 = verifyNot checkSpacefulnessCfg "declare -i s; s='x + y'; x=$s; echo $x" +prop_checkSpacefulnessCfg65 = verifyNot checkSpacefulnessCfg "f() { s=$?; echo $s; }; f" +prop_checkSpacefulnessCfg66 = verifyNot checkSpacefulnessCfg "f() { s=$?; echo $s; }" checkSpacefulnessCfg = checkSpacefulnessCfg' True checkVerboseSpacefulnessCfg = checkSpacefulnessCfg' False diff --git a/src/ShellCheck/CFGAnalysis.hs b/src/ShellCheck/CFGAnalysis.hs index 634d354..7b270a8 100644 --- a/src/ShellCheck/CFGAnalysis.hs +++ b/src/ShellCheck/CFGAnalysis.hs @@ -1300,8 +1300,7 @@ dataflow ctx entry = do outgoing = map snd outgoingL isRegular = ((== CFEFlow) . fst) -runRoot ctx entry exit = do - let env = createEnvironmentState +runRoot ctx env entry exit = do writeSTRef (cInput ctx) $ env writeSTRef (cOutput ctx) $ env writeSTRef (cNode ctx) $ entry @@ -1321,9 +1320,10 @@ analyzeControlFlow params t = runST $ f cfg entry exit where f cfg entry exit = do + let env = createEnvironmentState ctx <- newCtx $ cfGraph cfg -- Do a dataflow analysis starting on the root node - exitState <- runRoot ctx entry exit + exitState <- runRoot ctx env entry exit -- All nodes we've touched invocations <- readSTRef $ cInvocations ctx @@ -1336,7 +1336,7 @@ analyzeControlFlow params t = let uninvoked = M.difference declaredFunctions invokedNodes let stragglerInput = - exitState { + (env `patchState` exitState) { -- We don't want `die() { exit $?; }; echo "Sourced"` to assume $? is always echo sExitCodes = Nothing } From 581981ba7696e642ce4ffa57d6cc2a585ad631d5 Mon Sep 17 00:00:00 2001 From: Christian Nassif-Haynes Date: Sat, 24 Sep 2022 07:20:48 +1000 Subject: [PATCH 275/459] Suppress SC2311 with `set -o posix` --- src/ShellCheck/Analytics.hs | 1 + src/ShellCheck/AnalyzerLib.hs | 21 +++++++++------------ 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index e9ea36a..9d924f6 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -4686,6 +4686,7 @@ prop_checkSetESuppressed15 = verifyTree checkSetESuppressed "set -e; f(){ :; prop_checkSetESuppressed16 = verifyTree checkSetESuppressed "set -e; f(){ :; }; until set -e; f; do :; done" prop_checkSetESuppressed17 = verifyNotTree checkSetESuppressed "set -e; f(){ :; }; g(){ :; }; g f" prop_checkSetESuppressed18 = verifyNotTree checkSetESuppressed "set -e; shopt -s inherit_errexit; f(){ :; }; x=$(f)" +prop_checkSetESuppressed19 = verifyNotTree checkSetESuppressed "set -e; set -o posix; f(){ :; }; x=$(f)" checkSetESuppressed params t = if hasSetE params then runNodeAnalysis checkNode params t else [] where diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 88da89e..444c751 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -203,22 +203,22 @@ makeParameters spec = params hasSetE = containsSetE root, hasLastpipe = case shellType params of - Bash -> containsLastpipe root + Bash -> isOptionSet "lastpipe" root Dash -> False Sh -> False Ksh -> True, hasInheritErrexit = case shellType params of - Bash -> containsInheritErrexit root + Bash -> isOptionSet "inherit_errexit" root Dash -> True Sh -> True Ksh -> False, hasPipefail = case shellType params of - Bash -> containsPipefail root + Bash -> isOptionSet "pipefail" root Dash -> True Sh -> True - Ksh -> containsPipefail root, + Ksh -> isOptionSet "pipefail" root, shellTypeSpecified = isJust (asShellType spec) || isJust (asFallbackShell spec), idMap = getTokenMap root, parentMap = getParentTree root, @@ -247,13 +247,14 @@ containsSetE root = isNothing $ doAnalysis (guard . not . isSetE) root _ -> False re = mkRegex "[[:space:]]-[^-]*e" -containsPipefail root = isNothing $ doAnalysis (guard . not . isPipefail) root + +containsSetOption opt root = isNothing $ doAnalysis (guard . not . isPipefail) root where isPipefail t = case t of T_SimpleCommand {} -> t `isUnqualifiedCommand` "set" && - ("pipefail" `elem` oversimplify t || + (opt `elem` oversimplify t || "o" `elem` map snd (getAllFlags t)) _ -> False @@ -267,12 +268,8 @@ containsShopt shopt root = (shopt `elem` oversimplify t) _ -> False --- Does this script mention 'shopt -s inherit_errexit' anywhere? -containsInheritErrexit = containsShopt "inherit_errexit" - --- Does this script mention 'shopt -s lastpipe' anywhere? --- Also used as a hack. -containsLastpipe = containsShopt "lastpipe" +-- Does this script mention 'shopt -s $opt' or 'set -o $opt' anywhere? +isOptionSet opt root = containsShopt opt root || containsSetOption opt root prop_determineShell0 = determineShellTest "#!/bin/sh" == Sh From ef5f9a7af5e2f8287c535a45bba352ea193aab4b Mon Sep 17 00:00:00 2001 From: Christian Nassif-Haynes Date: Sun, 25 Sep 2022 03:04:20 +1000 Subject: [PATCH 276/459] Add `mapfile` to harmless commands for SC2094 --- src/ShellCheck/Analytics.hs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index e9ea36a..7237462 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -807,6 +807,7 @@ prop_checkRedirectToSame6 = verifyNot checkRedirectToSame "echo foo > foo" prop_checkRedirectToSame7 = verifyNot checkRedirectToSame "sed 's/foo/bar/g' file | sponge file" prop_checkRedirectToSame8 = verifyNot checkRedirectToSame "while read -r line; do _=\"$fname\"; done <\"$fname\"" prop_checkRedirectToSame9 = verifyNot checkRedirectToSame "while read -r line; do cat < \"$fname\"; done <\"$fname\"" +prop_checkRedirectToSame10 = verifyNot checkRedirectToSame "mapfile -t foo (mapM_ (\x -> doAnalysis (checkOccurrences x) l) (getAllRedirs list))) list where @@ -852,7 +853,7 @@ checkRedirectToSame params s@(T_Pipeline _ _ list) = isHarmlessCommand arg = fromMaybe False $ do cmd <- getClosestCommand (parentMap params) arg name <- getCommandBasename cmd - return $ name `elem` ["echo", "printf", "sponge"] + return $ name `elem` ["echo", "mapfile", "printf", "sponge"] containsAssignment arg = fromMaybe False $ do cmd <- getClosestCommand (parentMap params) arg return $ isAssignment cmd From 128351f5ef002be2ca3d2b2e3e2859c2c6c84e9a Mon Sep 17 00:00:00 2001 From: Peter Oliver Date: Fri, 7 Oct 2022 17:02:31 +0100 Subject: [PATCH 277/459] Permit colon after exec MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ShellCheck throws warning SC2093 when a script contains commands that could never be executed because they are after an `exec`. Command `:` does nothing, so add it to the list of commands that don’t trigger this warning. --- src/ShellCheck/Analytics.hs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index aa99934..ba2379d 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1876,6 +1876,7 @@ prop_checkSpuriousExec7 = verifyNot checkSpuriousExec "exec file; echo failed; e prop_checkSpuriousExec8 = verifyNot checkSpuriousExec "exec {origout}>&1- >tmp.log 2>&1; bar" prop_checkSpuriousExec9 = verify checkSpuriousExec "for file in rc.d/*; do exec \"$file\"; done" prop_checkSpuriousExec10 = verifyNot checkSpuriousExec "exec file; r=$?; printf >&2 'failed\n'; return $r" +prop_checkSpuriousExec11 = verifyNot checkSpuriousExec "exec file; :" checkSpuriousExec _ = doLists where doLists (T_Script _ _ cmds) = doList cmds False @@ -1891,7 +1892,7 @@ checkSpuriousExec _ = doLists stripCleanup = reverse . dropWhile cleanup . reverse cleanup (T_Pipeline _ _ [cmd]) = - isCommandMatch cmd (`elem` ["echo", "exit", "printf", "return"]) + isCommandMatch cmd (`elem` [":", "echo", "exit", "printf", "return"]) || isAssignment cmd cleanup _ = False From 43aca62ca7ffa84a406623d81fd67758b577bfa5 Mon Sep 17 00:00:00 2001 From: Christian Nassif-Haynes Date: Sun, 9 Oct 2022 07:59:05 +1100 Subject: [PATCH 278/459] Fix false positive for SC2312 when using `time` --- src/ShellCheck/Analytics.hs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index aa99934..bf77c9c 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -4765,8 +4765,12 @@ prop_checkExtraMaskedReturns32 = verifyNotTree checkExtraMaskedReturns "false < prop_checkExtraMaskedReturns33 = verifyNotTree checkExtraMaskedReturns "{ false || true; } | true" prop_checkExtraMaskedReturns34 = verifyNotTree checkExtraMaskedReturns "{ false || :; } | true" prop_checkExtraMaskedReturns35 = verifyTree checkExtraMaskedReturns "f() { local -r x=$(false); }" +prop_checkExtraMaskedReturns36 = verifyNotTree checkExtraMaskedReturns "time false" +prop_checkExtraMaskedReturns37 = verifyNotTree checkExtraMaskedReturns "time $(time false)" +prop_checkExtraMaskedReturns38 = verifyTree checkExtraMaskedReturns "x=$(time time time false) time $(time false)" -checkExtraMaskedReturns params t = runNodeAnalysis findMaskingNodes params t +checkExtraMaskedReturns params t = + runNodeAnalysis findMaskingNodes params (removeTransparentCommands t) where findMaskingNodes _ (T_Arithmetic _ list) = findMaskedNodesInList [list] findMaskingNodes _ (T_Array _ list) = findMaskedNodesInList $ allButLastSimpleCommands list @@ -4799,6 +4803,13 @@ checkExtraMaskedReturns params t = runNodeAnalysis findMaskingNodes params t where simpleCommands = filter containsSimpleCommand cmds + removeTransparentCommands t = + doTransform go t + where + go cmd@(T_SimpleCommand id assigns (_:args)) | isTransparentCommand cmd + = T_SimpleCommand id assigns args + go t = t + inform t = info (getId t) 2312 ("Consider invoking this command " ++ "separately to avoid masking its return value (or use '|| true' " ++ "to ignore).") @@ -4831,6 +4842,10 @@ checkExtraMaskedReturns params t = runNodeAnalysis findMaskingNodes params t ,"shopt" ] + isTransparentCommand t = fromMaybe False $ do + basename <- getCommandBasename t + return $ basename == "time" + parentChildPairs t = go $ parents params t where go (child:parent:rest) = (parent, child):go (parent:rest) From 81c2ecaccb47ced29e9e596f0d53b3068eff6811 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 11 Oct 2022 19:40:29 -0700 Subject: [PATCH 279/459] Remove true/false from SC2216/SC2217 (fixes #2603) --- src/ShellCheck/Data.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Data.hs b/src/ShellCheck/Data.hs index 7d6f5b4..550ff87 100644 --- a/src/ShellCheck/Data.hs +++ b/src/ShellCheck/Data.hs @@ -121,10 +121,10 @@ commonCommands = [ nonReadingCommands = [ "alias", "basename", "bg", "cal", "cd", "chgrp", "chmod", "chown", - "cp", "du", "echo", "export", "false", "fg", "fuser", "getconf", + "cp", "du", "echo", "export", "fg", "fuser", "getconf", "getopt", "getopts", "ipcrm", "ipcs", "jobs", "kill", "ln", "ls", "locale", "mv", "printf", "ps", "pwd", "renice", "rm", "rmdir", - "set", "sleep", "touch", "trap", "true", "ulimit", "unalias", "uname" + "set", "sleep", "touch", "trap", "ulimit", "unalias", "uname" ] sampleWords = [ From fa7943ac0e79dc3ac94bede40c55b8f407a699d6 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 11 Oct 2022 20:10:34 -0700 Subject: [PATCH 280/459] Revert "Add employer mandated disclaimer" This reverts commit 5202072a3439935fbc5a9b92fe66833633f63437. --- LICENSE | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/LICENSE b/LICENSE index 0df6056..f288702 100644 --- a/LICENSE +++ b/LICENSE @@ -1,13 +1,3 @@ -Employer mandated disclaimer: - - I am providing code in the repository to you under an open source license. - Because this is my personal repository, the license you receive to my code is - from me and other individual contributors, and not my employer (Facebook). - - - Vidar "koala_man" Holen - ----- - GNU GENERAL PUBLIC LICENSE Version 3, 29 June 2007 From a524929b6920ed4950b5bf5a97c57a40dd9b09f5 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 12 Oct 2022 20:21:48 -0700 Subject: [PATCH 281/459] Remove outdated test --- src/ShellCheck/Analytics.hs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index bf77c9c..e8bc4b9 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -3575,7 +3575,6 @@ prop_checkPipeToNowhere4 = verify checkPipeToNowhere "printf 'Lol' << eof\nlol\n prop_checkPipeToNowhere5 = verifyNot checkPipeToNowhere "echo foo | xargs du" prop_checkPipeToNowhere6 = verifyNot checkPipeToNowhere "ls | echo $(cat)" prop_checkPipeToNowhere7 = verifyNot checkPipeToNowhere "echo foo | var=$(cat) ls" -prop_checkPipeToNowhere8 = verify checkPipeToNowhere "foo | true" prop_checkPipeToNowhere9 = verifyNot checkPipeToNowhere "mv -i f . < /dev/stdin" prop_checkPipeToNowhere10 = verify checkPipeToNowhere "ls > file | grep foo" prop_checkPipeToNowhere11 = verify checkPipeToNowhere "ls | grep foo < file" From 14056a7f3a5917cba81582b33624948e83d7a50d Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Wed, 12 Oct 2022 20:20:59 -0700 Subject: [PATCH 282/459] Don't suggest pgrep for `ps -p .. | grep` (fixes #2597) --- src/ShellCheck/Analytics.hs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index e8bc4b9..77c527f 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -550,6 +550,7 @@ prop_checkPipePitfalls19 = verifyNot checkPipePitfalls "foo | grep -A2 bar | wc prop_checkPipePitfalls20 = verifyNot checkPipePitfalls "foo | grep -B999 bar | wc -l" prop_checkPipePitfalls21 = verifyNot checkPipePitfalls "foo | grep --after-context 999 bar | wc -l" prop_checkPipePitfalls22 = verifyNot checkPipePitfalls "foo | grep -B 1 --after-context 999 bar | wc -l" +prop_checkPipePitfalls23 = verifyNot checkPipePitfalls "ps -o pid,args -p $(pgrep java) | grep -F net.shellcheck.Test" checkPipePitfalls _ (T_Pipeline id _ commands) = do for ["find", "xargs"] $ \(find:xargs:_) -> @@ -563,8 +564,15 @@ checkPipePitfalls _ (T_Pipeline id _ commands) = do ]) $ warn (getId find) 2038 "Use -print0/-0 or -exec + to allow for non-alphanumeric filenames." - for' ["ps", "grep"] $ - \x -> info x 2009 "Consider using pgrep instead of grepping ps output." + for ["ps", "grep"] $ + \(ps:grep:_) -> + let + psFlags = maybe [] (map snd . getAllFlags) $ getCommand ps + in + -- There are many ways to specify a pid: 1, -1, p 1, wup 1, -q 1, -p 1, --pid 1. + -- For simplicity we only deal with the most canonical looking flags: + unless (any (`elem` ["p", "pid", "q", "quick-pid"]) psFlags) $ + info (getId ps) 2009 "Consider using pgrep instead of grepping ps output." for ["grep", "wc"] $ \(grep:wc:_) -> @@ -782,6 +790,7 @@ prop_checkUnquotedExpansions7 = verifyNot checkUnquotedExpansions "cat << foo\n$ prop_checkUnquotedExpansions8 = verifyNot checkUnquotedExpansions "set -- $(seq 1 4)" prop_checkUnquotedExpansions9 = verifyNot checkUnquotedExpansions "echo foo `# inline comment`" prop_checkUnquotedExpansions10 = verify checkUnquotedExpansions "#!/bin/sh\nexport var=$(val)" +prop_checkUnquotedExpansions11 = verifyNot checkUnquotedExpansions "ps -p $(pgrep foo)" checkUnquotedExpansions params = check where @@ -795,7 +804,7 @@ checkUnquotedExpansions params = warn (getId t) 2046 "Quote this to prevent word splitting." shouldBeSplit t = - getCommandNameFromExpansion t == Just "seq" + getCommandNameFromExpansion t `elem` [Just "seq", Just "pgrep"] prop_checkRedirectToSame = verify checkRedirectToSame "cat foo > foo" From d9c9e60fb0064a45188dd93993ad86253ac5a0b3 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 13 Oct 2022 19:46:15 -0700 Subject: [PATCH 283/459] Allow arbitrary bats @test names (fixes #2587) --- src/ShellCheck/AST.hs | 2 +- src/ShellCheck/Parser.hs | 19 ++++++++++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/AST.hs b/src/ShellCheck/AST.hs index ca5007a..5c20416 100644 --- a/src/ShellCheck/AST.hs +++ b/src/ShellCheck/AST.hs @@ -142,7 +142,7 @@ data InnerToken t = | Inner_T_CoProcBody t | Inner_T_Include t | Inner_T_SourceCommand t t - | Inner_T_BatsTest t t + | Inner_T_BatsTest String t deriving (Show, Eq, Functor, Foldable, Traversable) data Annotation = diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index aeaf703..969f4b7 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -2500,16 +2500,29 @@ readBraceGroup = called "brace group" $ do spacing return $ T_BraceGroup id list -prop_readBatsTest = isOk readBatsTest "@test 'can parse' {\n true\n}" +prop_readBatsTest1 = isOk readBatsTest "@test 'can parse' {\n true\n}" +prop_readBatsTest2 = isOk readBatsTest "@test random text !(@*$Y&! {\n true\n}" +prop_readBatsTest3 = isOk readBatsTest "@test foo { bar { baz {\n true\n}" +prop_readBatsTest4 = isNotOk readBatsTest "@test foo \n{\n true\n}" readBatsTest = called "bats @test" $ do start <- startSpan - try $ string "@test" + try $ string "@test " spacing - name <- readNormalWord + name <- readBatsName spacing test <- readBraceGroup id <- endSpan start return $ T_BatsTest id name test + where + readBatsName = do + line <- try . lookAhead $ many1 $ noneOf "\n" + let name = reverse $ f $ reverse line + string name + + -- We want everything before the last " {" in a string, so we find everything after "{ " in its reverse + f ('{':' ':rest) = dropWhile isSpace rest + f (a:rest) = f rest + f [] = "" prop_readWhileClause = isOk readWhileClause "while [[ -e foo ]]; do sleep 1; done" readWhileClause = called "while loop" $ do From b770984dfcfb90d74b2ca7c1ad11cd35bb45d45e Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 13 Oct 2022 21:04:38 -0700 Subject: [PATCH 284/459] Try to parse the inside of traps (fixes #2584) --- src/ShellCheck/Parser.hs | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 969f4b7..837735a 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -2112,6 +2112,7 @@ prop_readSimpleCommand11 = isOk readSimpleCommand "/\\* foo" prop_readSimpleCommand12 = isWarning readSimpleCommand "elsif foo" prop_readSimpleCommand13 = isWarning readSimpleCommand "ElseIf foo" prop_readSimpleCommand14 = isWarning readSimpleCommand "elseif[$i==2]" +prop_readSimpleCommand15 = isWarning readSimpleCommand "trap 'foo\"bar' INT" readSimpleCommand = called "simple command" $ do prefix <- option [] readCmdPrefix skipAnnotationAndWarn @@ -2141,9 +2142,12 @@ readSimpleCommand = called "simple command" $ do id2 <- getNewIdFor id1 let result = makeSimpleCommand id1 id2 prefix [cmd] suffix - if isCommand ["source", "."] cmd - then readSource result - else return result + case () of + _ | isCommand ["source", "."] cmd -> readSource result + _ | isCommand ["trap"] cmd -> do + syntaxCheckTrap result + return result + _ -> return result where isCommand strings (T_NormalWord _ [T_Literal _ s]) = s `elem` strings isCommand _ _ = False @@ -2163,6 +2167,17 @@ readSimpleCommand = called "simple command" $ do parseProblemAtId (getId cmd) ErrorC 1131 "Use 'elif' to start another branch." _ -> return () + syntaxCheckTrap cmd = + case cmd of + (T_Redirecting _ _ (T_SimpleCommand _ _ (cmd:arg:_))) -> checkArg arg (getLiteralString arg) + _ -> return () + where + checkArg _ Nothing = return () + checkArg arg (Just ('-':_)) = return () + checkArg arg (Just str) = do + (start,end) <- getSpanForId (getId arg) + subParse start (tryWithErrors (readCompoundListOrEmpty >> verifyEof) <|> return ()) str + commentWarning id = parseProblemAtId id ErrorC 1127 "Was this intended as a comment? Use # in sh." From 86e2b76730bef5ec509828d138c67d6b49b89b7f Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 29 Oct 2022 12:50:07 -0700 Subject: [PATCH 285/459] Improve SC1059 error message --- src/ShellCheck/Parser.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 837735a..dd0f0f0 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -2566,7 +2566,7 @@ readDoGroup kwId = do parseProblem ErrorC 1058 "Expected 'do'." return "Expected 'do'" - acceptButWarn g_Semi ErrorC 1059 "No semicolons directly after 'do'." + acceptButWarn g_Semi ErrorC 1059 "Semicolon is not allowed directly after 'do'. You can just delete it." allspacing optional (do From 84d8530f14e35ff3fb7688fb11935b60b02deaf0 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 29 Oct 2022 12:50:37 -0700 Subject: [PATCH 286/459] Add SVG logo --- doc/shellcheck_logo.svg | 294 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 294 insertions(+) create mode 100644 doc/shellcheck_logo.svg diff --git a/doc/shellcheck_logo.svg b/doc/shellcheck_logo.svg new file mode 100644 index 0000000..836aa63 --- /dev/null +++ b/doc/shellcheck_logo.svg @@ -0,0 +1,294 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 3342902d9a354c71ba72a939af82d535f043d079 Mon Sep 17 00:00:00 2001 From: ArenM Date: Thu, 17 Nov 2022 18:06:10 -0500 Subject: [PATCH 287/459] Warn about 'read' without a variable in POSIX sh Dash throws an error if the read command isn't supplied a variable name. --- src/ShellCheck/Checks/ShellSupport.hs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 30a19b9..eda6882 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -184,6 +184,10 @@ prop_checkBashisms96 = verifyNot checkBashisms "#!/bin/dash\necho $_" prop_checkBashisms97 = verify checkBashisms "#!/bin/sh\necho ${var,}" prop_checkBashisms98 = verify checkBashisms "#!/bin/sh\necho ${var^^}" prop_checkBashisms99 = verify checkBashisms "#!/bin/dash\necho [^f]oo" +prop_checkBashisms100 = verify checkBashisms "read -r" +prop_checkBashisms101 = verify checkBashisms "read" +prop_checkBashisms102 = verifyNot checkBashisms "read -r foo" +prop_checkBashisms103 = verifyNot checkBashisms "read foo" checkBashisms = ForShell [Sh, Dash] $ \t -> do params <- ask kludge params t @@ -284,6 +288,13 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do argString = concat $ oversimplify arg flagRegex = mkRegex "^-[eEsn]+$" + bashism t@(T_SimpleCommand _ _ (cmd:args)) + | t `isCommand` "read" && length (onlyNames args) == 0 = + warnMsg (getId cmd) 3061 "read without a variable is" + where + notFlag arg = head (concat $ oversimplify arg) /= '-' + onlyNames = filter (notFlag) + bashism t@(T_SimpleCommand _ _ (cmd:arg:_)) | getLiteralString cmd == Just "exec" && "-" `isPrefixOf` concat (oversimplify arg) = warnMsg (getId arg) 3038 "exec flags are" From 2a16a4e8c18745887ca33c545854bdcb3097fede Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 10 Dec 2022 10:46:49 -0800 Subject: [PATCH 288/459] Add missing imports for later GHC versions --- ShellCheck.cabal | 9 ++++++--- shellcheck.hs | 2 ++ src/ShellCheck/AnalyzerLib.hs | 1 + src/ShellCheck/Fixer.hs | 1 + 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/ShellCheck.cabal b/ShellCheck.cabal index b22b5c8..abb32d0 100644 --- a/ShellCheck.cabal +++ b/ShellCheck.cabal @@ -54,11 +54,12 @@ library Diff >= 0.2.0, directory >= 1.2.3.0, fgl, - mtl >= 2.2.1, filepath, + mtl >= 2.2.1, parsec, - regex-tdfa, QuickCheck >= 2.7.4, + regex-tdfa, + transformers, -- When cabal supports it, move this to setup-depends: process exposed-modules: @@ -112,6 +113,7 @@ executable shellcheck parsec >= 3.0, QuickCheck >= 2.7.4, regex-tdfa, + transformers, ShellCheck default-language: Haskell98 main-is: shellcheck.hs @@ -128,11 +130,12 @@ test-suite test-shellcheck Diff >= 0.2.0, directory >= 1.2.3.0, fgl, - mtl >= 2.2.1, filepath, + mtl >= 2.2.1, parsec, QuickCheck >= 2.7.4, regex-tdfa, + transformers, ShellCheck default-language: Haskell98 main-is: test/shellcheck.hs diff --git a/shellcheck.hs b/shellcheck.hs index a525251..4e8a155 100644 --- a/shellcheck.hs +++ b/shellcheck.hs @@ -34,6 +34,8 @@ import qualified ShellCheck.Formatter.Quiet import Control.Exception import Control.Monad +import Control.Monad.IO.Class +import Control.Monad.Trans.Class import Control.Monad.Except import Data.Bits import Data.Char diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 444c751..47ea91d 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -32,6 +32,7 @@ import ShellCheck.Regex import Control.Arrow (first) import Control.DeepSeq +import Control.Monad import Control.Monad.Identity import Control.Monad.RWS import Control.Monad.State diff --git a/src/ShellCheck/Fixer.hs b/src/ShellCheck/Fixer.hs index 358dec9..0d3c8f4 100644 --- a/src/ShellCheck/Fixer.hs +++ b/src/ShellCheck/Fixer.hs @@ -23,6 +23,7 @@ module ShellCheck.Fixer (applyFix, removeTabStops, mapPositions, Ranged(..), run import ShellCheck.Interface import ShellCheck.Prelude +import Control.Monad import Control.Monad.State import Data.Array import Data.List From 495e34d10179715e8a675ca3e721b47757c9dc0f Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 11 Dec 2022 14:18:47 -0800 Subject: [PATCH 289/459] Add missing Semigroup import for older GHC --- src/ShellCheck/Prelude.hs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/ShellCheck/Prelude.hs b/src/ShellCheck/Prelude.hs index 7e9011b..7610c46 100644 --- a/src/ShellCheck/Prelude.hs +++ b/src/ShellCheck/Prelude.hs @@ -21,6 +21,9 @@ -- Generic basic utility functions module ShellCheck.Prelude where +import Data.Semigroup + + -- Get element 0 or a default. Like `head` but safe. headOrDefault _ (a:_) = a headOrDefault def _ = def From 74b1745a1998c5b0a203a5d3c67dade7997ad1b2 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 11 Dec 2022 14:48:00 -0800 Subject: [PATCH 290/459] Fix compiler error on some GHC versions Fixes the following error: src/ShellCheck/CFGAnalysis.hs:1394:40: error: * Couldn't match expected type `[S.Set a]' with actual type `M.Map String FunctionValue' * In the second argument of `($)', namely `mapStorage $ sFunctionTargets state' In the expression: S.unions $ mapStorage $ sFunctionTargets state In an equation for `declaredFuncs': declaredFuncs = S.unions $ mapStorage $ sFunctionTargets state * Relevant bindings include declaredFuncs :: S.Set a (bound at src/ShellCheck/CFGAnalysis.hs:1394:13) --- src/ShellCheck/CFGAnalysis.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/CFGAnalysis.hs b/src/ShellCheck/CFGAnalysis.hs index 7b270a8..cac913e 100644 --- a/src/ShellCheck/CFGAnalysis.hs +++ b/src/ShellCheck/CFGAnalysis.hs @@ -1391,7 +1391,7 @@ analyzeControlFlow params t = getFunctionTargets :: InternalState -> M.Map Node FunctionDefinition getFunctionTargets state = let - declaredFuncs = S.unions $ mapStorage $ sFunctionTargets state + declaredFuncs = S.unions $ M.elems $ mapStorage $ sFunctionTargets state getFunc d = case d of FunctionDefinition _ entry _ -> Just (entry, d) From 3cae6cd6abe16b18a1c95598561d5afca1135cec Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 11 Dec 2022 15:05:33 -0800 Subject: [PATCH 291/459] Allow building on deepseq < 1.4.2.0 --- src/ShellCheck/CFGAnalysis.hs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/ShellCheck/CFGAnalysis.hs b/src/ShellCheck/CFGAnalysis.hs index cac913e..4e36cf5 100644 --- a/src/ShellCheck/CFGAnalysis.hs +++ b/src/ShellCheck/CFGAnalysis.hs @@ -20,6 +20,7 @@ {-# LANGUAGE TemplateHaskell #-} {-# LANGUAGE RankNTypes #-} {-# LANGUAGE DeriveAnyClass, DeriveGeneric #-} +{-# LANGUAGE CPP #-} {- Data Flow Analysis on a Control Flow Graph. @@ -433,6 +434,13 @@ data StackEntry s = StackEntry { } deriving (Eq, Generic, NFData) +#if MIN_VERSION_deepseq(1,4,2) +-- Our deepseq already has a STRef instance +#else +-- Older deepseq (for GHC < 8) lacks this instance +instance NFData (STRef s a) where + rnf = (`seq` ()) +#endif -- Overwrite a base state with the contents of a diff state -- This is unrelated to join/merge. From 985ca2530d475f6bf93fa81d31cc220fccddeea8 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 11 Dec 2022 16:34:29 -0800 Subject: [PATCH 292/459] Add Docker testing for older and newer Ubuntu versions --- test/buildtest | 3 ++- test/distrotest | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/test/buildtest b/test/buildtest index 1d194fc..469539b 100755 --- a/test/buildtest +++ b/test/buildtest @@ -22,7 +22,8 @@ fi cabal install --dependencies-only --enable-tests "${flags[@]}" || cabal install --dependencies-only "${flags[@]}" || - die "can't install dependencies" + cabal install --dependencies-only --max-backjumps -1 "${flags[@]}" || + die "can't install dependencies" cabal configure --enable-tests "${flags[@]}" || die "configure failed" cabal build || diff --git a/test/distrotest b/test/distrotest index 464768c..e1711ea 100755 --- a/test/distrotest +++ b/test/distrotest @@ -67,7 +67,10 @@ fedora:latest dnf install -y cabal-install ghc-template-haskell-devel fi archlinux:latest pacman -S -y --noconfirm cabal-install ghc-static base-devel # Ubuntu LTS +ubuntu:22.04 apt-get update && apt-get install -y cabal-install ubuntu:20.04 apt-get update && apt-get install -y cabal-install +ubuntu:18.04 apt-get update && apt-get install -y cabal-install +ubuntu:16.04 apt-get update && apt-get install -y cabal-install # Stack on Ubuntu LTS ubuntu:20.04 set -e; apt-get update && apt-get install -y curl && curl -sSL https://get.haskellstack.org/ | sh -s - -f && cd /mnt && exec test/stacktest From 8754c21244ada70c070763644044fa166387f708 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 11 Dec 2022 16:37:49 -0800 Subject: [PATCH 293/459] Avoid $ trigger TH --- src/ShellCheck/Formatter/CheckStyle.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Formatter/CheckStyle.hs b/src/ShellCheck/Formatter/CheckStyle.hs index c79ac21..6ad6c9c 100644 --- a/src/ShellCheck/Formatter/CheckStyle.hs +++ b/src/ShellCheck/Formatter/CheckStyle.hs @@ -88,7 +88,7 @@ outputError file error = putStrLn $ concat [ attr s v = concat [ s, "='", escape v, "' " ] escape = concatMap escape' escape' c = if isOk c then [c] else "&#" ++ show (ord c) ++ ";" -isOk x = any ($x) [isAsciiUpper, isAsciiLower, isDigit, (`elem` " ./")] +isOk x = any ($ x) [isAsciiUpper, isAsciiLower, isDigit, (`elem` " ./")] severity "error" = "error" severity "warning" = "warning" From a7c5be93dcbd4c219615e44030073158df4426e5 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 11 Dec 2022 12:29:05 -0800 Subject: [PATCH 294/459] Tighten bounds on packages --- ShellCheck.cabal | 61 ++++++++++++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 28 deletions(-) diff --git a/ShellCheck.cabal b/ShellCheck.cabal index abb32d0..dab588c 100644 --- a/ShellCheck.cabal +++ b/ShellCheck.cabal @@ -45,21 +45,26 @@ library build-depends: semigroups build-depends: - aeson, - array, - base >= 4.8.0.0 && < 5, - bytestring, - containers >= 0.5, - deepseq >= 1.4.0.0, - Diff >= 0.2.0, - directory >= 1.2.3.0, - fgl, - filepath, - mtl >= 2.2.1, - parsec, - QuickCheck >= 2.7.4, - regex-tdfa, - transformers, + -- The lower bounds are based on GHC 7.10.3 + -- The upper bounds are based on GHC 9.4.3 + aeson >= 1.4.0 && < 2.2, + array >= 0.5.1 && < 0.6, + base >= 4.8.0.0 && < 5, + bytestring >= 0.10.6 && < 0.12, + containers >= 0.5.6 && < 0.7, + deepseq >= 1.4.1 && < 1.5, + Diff >= 0.4.0 && < 0.5, + fgl >= 5.7.0 && < 5.9, + filepath >= 1.4.0 && < 1.5, + mtl >= 2.2.2 && < 2.3, + parsec >= 3.1.14 && < 3.2, + QuickCheck >= 2.14.2 && < 2.15, + regex-tdfa >= 1.2.0 && < 1.4, + transformers >= 0.4.2 && < 0.6, + + -- getXdgDirectory from 1.2.3.0 + directory >= 1.2.3 && < 1.4, + -- When cabal supports it, move this to setup-depends: process exposed-modules: @@ -101,17 +106,17 @@ executable shellcheck build-depends: aeson, array, - base >= 4 && < 5, + base, bytestring, containers, - deepseq >= 1.4.0.0, - Diff >= 0.2.0, - directory >= 1.2.3.0, + deepseq, + Diff, + directory, fgl, - mtl >= 2.2.1, + mtl, filepath, - parsec >= 3.0, - QuickCheck >= 2.7.4, + parsec, + QuickCheck, regex-tdfa, transformers, ShellCheck @@ -123,17 +128,17 @@ test-suite test-shellcheck build-depends: aeson, array, - base >= 4 && < 5, + base, bytestring, containers, - deepseq >= 1.4.0.0, - Diff >= 0.2.0, - directory >= 1.2.3.0, + deepseq, + Diff, + directory, fgl, filepath, - mtl >= 2.2.1, + mtl, parsec, - QuickCheck >= 2.7.4, + QuickCheck, regex-tdfa, transformers, ShellCheck From 7cfcf6db8a3d5a15cfd361293bbbe2d38d473c5e Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 11 Dec 2022 19:22:42 -0800 Subject: [PATCH 295/459] Fix stack build --- test/distrotest | 2 +- test/stacktest | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/distrotest b/test/distrotest index e1711ea..53a40a7 100755 --- a/test/distrotest +++ b/test/distrotest @@ -73,7 +73,7 @@ ubuntu:18.04 apt-get update && apt-get install -y cabal-install ubuntu:16.04 apt-get update && apt-get install -y cabal-install # Stack on Ubuntu LTS -ubuntu:20.04 set -e; apt-get update && apt-get install -y curl && curl -sSL https://get.haskellstack.org/ | sh -s - -f && cd /mnt && exec test/stacktest +ubuntu:22.04 set -e; apt-get update && apt-get install -y curl && curl -sSL https://get.haskellstack.org/ | sh -s - -f && cd /mnt && exec test/stacktest EOF exit "$final" diff --git a/test/stacktest b/test/stacktest index ae04f1b..9eb8d1e 100755 --- a/test/stacktest +++ b/test/stacktest @@ -3,7 +3,7 @@ # various resolvers. It's run via distrotest. resolvers=( - nightly-"$(date -d "3 days ago" +"%Y-%m-%d")" +# nightly-"$(date -d "3 days ago" +"%Y-%m-%d")" ) die() { echo "$*" >&2; exit 1; } From ae199edb680dd416790b7890fa52e64a11f2b4af Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 11 Dec 2022 20:50:33 -0800 Subject: [PATCH 296/459] Let distrotest fail fast when there remaining executables --- test/distrotest | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/distrotest b/test/distrotest index 53a40a7..4ad66f8 100755 --- a/test/distrotest +++ b/test/distrotest @@ -25,6 +25,13 @@ exit 0 echo "Deleting 'dist' and 'dist-newstyle'..." rm -rf dist dist-newstyle +execs=$(find . -name shellcheck) + +if [ -n "$execs" ] +then + die "Found unexpected executables. Remove and try again: $execs" +fi + log=$(mktemp) || die "Can't create temp file" date >> "$log" || die "Can't write to log" From 8c5fdc3522236767e9ce840b57da2a13b32eb4ed Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 12 Dec 2022 21:49:01 -0800 Subject: [PATCH 297/459] Update copyright years --- shellcheck.1.md | 2 +- src/ShellCheck/Analytics.hs | 2 +- src/ShellCheck/Analyzer.hs | 2 +- src/ShellCheck/AnalyzerLib.hs | 2 +- src/ShellCheck/Checker.hs | 2 +- src/ShellCheck/Checks/Commands.hs | 2 +- src/ShellCheck/Parser.hs | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/shellcheck.1.md b/shellcheck.1.md index c345a2b..9675e79 100644 --- a/shellcheck.1.md +++ b/shellcheck.1.md @@ -378,7 +378,7 @@ long list of wonderful contributors. # COPYRIGHT -Copyright 2012-2021, Vidar Holen and contributors. +Copyright 2012-2022, Vidar Holen and contributors. Licensed under the GNU General Public License version 3 or later, see https://gnu.org/licenses/gpl.html diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index f50510d..1f6d96d 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1,5 +1,5 @@ {- - Copyright 2012-2021 Vidar Holen + Copyright 2012-2022 Vidar Holen This file is part of ShellCheck. https://www.shellcheck.net diff --git a/src/ShellCheck/Analyzer.hs b/src/ShellCheck/Analyzer.hs index 06b6e53..53717ed 100644 --- a/src/ShellCheck/Analyzer.hs +++ b/src/ShellCheck/Analyzer.hs @@ -1,5 +1,5 @@ {- - Copyright 2012-2019 Vidar Holen + Copyright 2012-2022 Vidar Holen This file is part of ShellCheck. https://www.shellcheck.net diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 47ea91d..ca928fd 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -1,5 +1,5 @@ {- - Copyright 2012-2021 Vidar Holen + Copyright 2012-2022 Vidar Holen This file is part of ShellCheck. https://www.shellcheck.net diff --git a/src/ShellCheck/Checker.hs b/src/ShellCheck/Checker.hs index c8d2c39..b56be68 100644 --- a/src/ShellCheck/Checker.hs +++ b/src/ShellCheck/Checker.hs @@ -1,5 +1,5 @@ {- - Copyright 2012-2020 Vidar Holen + Copyright 2012-2022 Vidar Holen This file is part of ShellCheck. https://www.shellcheck.net diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index e97ecd6..691836f 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -1,5 +1,5 @@ {- - Copyright 2012-2021 Vidar Holen + Copyright 2012-2022 Vidar Holen This file is part of ShellCheck. https://www.shellcheck.net diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index dd0f0f0..7a50967 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -1,5 +1,5 @@ {- - Copyright 2012-2021 Vidar Holen + Copyright 2012-2022 Vidar Holen This file is part of ShellCheck. https://www.shellcheck.net From a526ee08290cc127bc1aa5a05e9b927af87b6ef3 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 12 Dec 2022 19:58:11 -0800 Subject: [PATCH 298/459] Stable version 0.9.0 This release is dedicated to Mindustry: the most fun you can have with open source (outside of shell scripting of course). --- CHANGELOG.md | 2 +- ShellCheck.cabal | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c363eb5..57951c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## Git +## v0.9.0 - 2022-12-12 ### Added - SC2316: Warn about 'local readonly foo' and similar (thanks, patrickxia!) - SC2317: Warn about unreachable commands diff --git a/ShellCheck.cabal b/ShellCheck.cabal index dab588c..1226588 100644 --- a/ShellCheck.cabal +++ b/ShellCheck.cabal @@ -1,5 +1,5 @@ Name: ShellCheck -Version: 0.8.0 +Version: 0.9.0 Synopsis: Shell script analysis tool License: GPL-3 License-file: LICENSE From 5a3eb89e385d3667ecda33cfe769571ffc9dd7a3 Mon Sep 17 00:00:00 2001 From: Samuel Lijin Date: Fri, 3 Feb 2023 09:17:47 -0800 Subject: [PATCH 299/459] Document Trunk Check integration Trunk Check is a universal linter which integrates with a wide variety of linters and formatters, `shellcheck` included. We're big fans of `shellcheck` and figured that you might find our tool to be interesting enough to include it in the integrations list. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 6f3e4a9..8cf3584 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,7 @@ Services and platforms that have ShellCheck pre-installed and ready to use: * [Code Factor](https://www.codefactor.io/) * [CircleCI](https://circleci.com) via the [ShellCheck Orb](https://circleci.com/orbs/registry/orb/circleci/shellcheck) * [Github](https://github.com/features/actions) (only Linux) +* [Trunk Check](https://trunk.io/products/check) (universal linter; [allows you to explicitly version your shellcheck install](https://github.com/trunk-io/plugins/blob/bcbb361dcdbe4619af51ea7db474d7fb87540d20/.trunk/trunk.yaml#L32)) via the [shellcheck plugin](https://github.com/trunk-io/plugins/blob/main/linters/shellcheck/plugin.yaml) Most other services, including [GitLab](https://about.gitlab.com/), let you install ShellCheck yourself, either through the system's package manager (see [Installing](#installing)), From 78dea1d4f93987c7e52df121f8062da3387d0f2e Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 4 Feb 2023 10:27:59 -0800 Subject: [PATCH 300/459] Update changelog from release --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 57951c8..fc1c0ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +## Git +### Added + +### Fixed + +### Changed + + ## v0.9.0 - 2022-12-12 ### Added - SC2316: Warn about 'local readonly foo' and similar (thanks, patrickxia!) From 2842ce97b88a9cd9551f5f42f67a97e17a523b53 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 4 Feb 2023 11:38:20 -0800 Subject: [PATCH 301/459] Remove fgl-5.8.1.0 as a dependency ShellCheck is temporarily broken by https://github.com/haskell/fgl/commit/c8f56c18242b0c3e916892c5db56a609ec396637 --- ShellCheck.cabal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ShellCheck.cabal b/ShellCheck.cabal index 1226588..8afebe1 100644 --- a/ShellCheck.cabal +++ b/ShellCheck.cabal @@ -54,7 +54,7 @@ library containers >= 0.5.6 && < 0.7, deepseq >= 1.4.1 && < 1.5, Diff >= 0.4.0 && < 0.5, - fgl >= 5.7.0 && < 5.9, + fgl >= 5.7.0 && < 5.8.1.0, filepath >= 1.4.0 && < 1.5, mtl >= 2.2.2 && < 2.3, parsec >= 3.1.14 && < 3.2, From c05380d518056189412e12128a8906b8ca6f6717 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 4 Feb 2023 13:19:27 -0800 Subject: [PATCH 302/459] Count CFEExit as control flow for the purposes of finding dominators --- src/ShellCheck/CFG.hs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/CFG.hs b/src/ShellCheck/CFG.hs index e0c6267..f882adc 100644 --- a/src/ShellCheck/CFG.hs +++ b/src/ShellCheck/CFG.hs @@ -192,7 +192,7 @@ buildGraph params root = base idToRange = M.fromList mapping - isRealEdge (from, to, edge) = case edge of CFEFlow -> True; _ -> False + isRealEdge (from, to, edge) = case edge of CFEFlow -> True; CFEExit -> True; _ -> False onlyRealEdges = filter isRealEdge edges (_, mainExit) = fromJust $ M.lookup (getId root) idToRange @@ -1301,7 +1301,10 @@ findPostDominators mainexit graph = asArray reversed = grev withExitEdges postDoms = dom reversed mainexit (_, maxNode) = nodeRange graph - asArray = array (0, maxNode) postDoms + -- Holes in the array cause "Exception: (Array.!): undefined array element" while + -- inspecting/debugging, so fill the array first and then update. + initializedArray = listArray (0, maxNode) $ repeat [] + asArray = initializedArray // postDoms return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) From b1ca3929e387446f3e3db023d716cf3787370437 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 4 Feb 2023 19:55:25 -0800 Subject: [PATCH 303/459] Upgrade cross-compilers to 9.2.5 to handle hashable-1.4.2.0 --- build/darwin.x86_64/Dockerfile | 11 +++++++---- build/linux.aarch64/Dockerfile | 14 ++++++++++---- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/build/darwin.x86_64/Dockerfile b/build/darwin.x86_64/Dockerfile index 9e33a82..a53245f 100644 --- a/build/darwin.x86_64/Dockerfile +++ b/build/darwin.x86_64/Dockerfile @@ -6,15 +6,18 @@ ENV TARGETNAME darwin.x86_64 # Build dependencies USER root ENV DEBIAN_FRONTEND noninteractive -RUN apt-get update && apt-get install -y ghc automake autoconf llvm curl +RUN sed -e 's/focal/kinetic/g' -i /etc/apt/sources.list +RUN apt-get update +RUN apt-get dist-upgrade -y +RUN apt-get install -y ghc automake autoconf llvm curl alex happy # Build GHC WORKDIR /ghc -RUN curl -L "https://downloads.haskell.org/~ghc/8.10.4/ghc-8.10.4-src.tar.xz" | tar xJ --strip-components=1 -RUN ./boot && ./configure --host x86_64-linux-gnu --build x86_64-linux-gnu --target "$TARGET" +RUN curl -L "https://downloads.haskell.org/~ghc/9.2.5/ghc-9.2.5-src.tar.xz" | tar xJ --strip-components=1 +RUN ./configure --host x86_64-linux-gnu --build x86_64-linux-gnu --target "$TARGET" RUN cp mk/flavours/quick-cross.mk mk/build.mk && make -j "$(nproc)" RUN make install -RUN curl -L "https://downloads.haskell.org/~cabal/cabal-install-3.2.0.0/cabal-install-3.2.0.0-x86_64-unknown-linux.tar.xz" | tar xJv -C /usr/local/bin +RUN curl -L "https://downloads.haskell.org/~cabal/cabal-install-3.9.0.0/cabal-install-3.9-x86_64-linux-alpine.tar.xz" | tar xJv -C /usr/local/bin # Due to an apparent cabal bug, we specify our options directly to cabal # It won't reuse caches if ghc-options are specified in ~/.cabal/config diff --git a/build/linux.aarch64/Dockerfile b/build/linux.aarch64/Dockerfile index 60537b3..d5320e9 100644 --- a/build/linux.aarch64/Dockerfile +++ b/build/linux.aarch64/Dockerfile @@ -6,19 +6,25 @@ ENV TARGETNAME linux.aarch64 # Build dependencies USER root ENV DEBIAN_FRONTEND noninteractive -RUN apt-get update && apt-get install -y ghc automake autoconf build-essential llvm curl qemu-user-static gcc-$TARGET + +# These deps are from 20.04, because GHC's compiler/llvm support moves slowly +RUN apt-get update && apt-get install -y llvm gcc-$TARGET + +# The rest are from 22.10 +RUN sed -e 's/focal/kinetic/g' -i /etc/apt/sources.list +RUN apt-get update && apt-get install -y ghc alex happy automake autoconf build-essential curl qemu-user-static # Build GHC WORKDIR /ghc -RUN curl -L "https://downloads.haskell.org/~ghc/8.10.4/ghc-8.10.4-src.tar.xz" | tar xJ --strip-components=1 +RUN curl -L "https://downloads.haskell.org/~ghc/9.2.5/ghc-9.2.5-src.tar.xz" | tar xJ --strip-components=1 RUN ./boot && ./configure --host x86_64-linux-gnu --build x86_64-linux-gnu --target "$TARGET" RUN cp mk/flavours/quick-cross.mk mk/build.mk && make -j "$(nproc)" RUN make install -RUN curl -L "https://downloads.haskell.org/~cabal/cabal-install-3.2.0.0/cabal-install-3.2.0.0-x86_64-unknown-linux.tar.xz" | tar xJv -C /usr/local/bin +RUN curl -L "https://downloads.haskell.org/~cabal/cabal-install-3.9.0.0/cabal-install-3.9-x86_64-linux-alpine.tar.xz" | tar xJv -C /usr/local/bin # Due to an apparent cabal bug, we specify our options directly to cabal # It won't reuse caches if ghc-options are specified in ~/.cabal/config -ENV CABALOPTS "--ghc-options;-split-sections -optc-Os -optc-Wl,--gc-sections;--with-ghc=$TARGET-ghc;--with-hc-pkg=$TARGET-ghc-pkg" +ENV CABALOPTS "--ghc-options;-split-sections -optc-Os -optc-Wl,--gc-sections -optc-fPIC;--with-ghc=$TARGET-ghc;--with-hc-pkg=$TARGET-ghc-pkg" # Prebuild the dependencies RUN cabal update && IFS=';' && cabal install --dependencies-only $CABALOPTS ShellCheck From e6e8ab0415f720afee5f426b6ca09757a4ed12ce Mon Sep 17 00:00:00 2001 From: Felipe Santos Date: Sun, 5 Feb 2023 11:13:07 -0300 Subject: [PATCH 304/459] Mention VS Code ShellCheck binaries distribution --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 6f3e4a9..a1c59a7 100644 --- a/README.md +++ b/README.md @@ -232,6 +232,8 @@ Alternatively, you can download pre-compiled binaries for the latest release her or see the [GitHub Releases](https://github.com/koalaman/shellcheck/releases) for other releases (including the [latest](https://github.com/koalaman/shellcheck/releases/tag/latest) meta-release for daily git builds). +You can also find pre-compiled binaries repackaged in `.tar.gz` format in the [VS Code ShellCheck Binaries](https://github.com/vscode-shellcheck/shellcheck-binaries/releases) repository. It also includes a pre-compiled binary for **Apple M1** processors. + Distro packages already come with a `man` page. If you are building from source, it can be installed with: ```console From 08b437974e871d82213545b9a079bcb794f7a58a Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 23 Apr 2023 16:47:49 -0700 Subject: [PATCH 305/459] Rewrite vscode-shellcheck blurb --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a1c59a7..3a839aa 100644 --- a/README.md +++ b/README.md @@ -232,7 +232,8 @@ Alternatively, you can download pre-compiled binaries for the latest release her or see the [GitHub Releases](https://github.com/koalaman/shellcheck/releases) for other releases (including the [latest](https://github.com/koalaman/shellcheck/releases/tag/latest) meta-release for daily git builds). -You can also find pre-compiled binaries repackaged in `.tar.gz` format in the [VS Code ShellCheck Binaries](https://github.com/vscode-shellcheck/shellcheck-binaries/releases) repository. It also includes a pre-compiled binary for **Apple M1** processors. +There are currently no official binaries for Apple Silicon, but third party builds are available via +[ShellCheck for Visual Studio Code](https://github.com/vscode-shellcheck/shellcheck-binaries/releases). Distro packages already come with a `man` page. If you are building from source, it can be installed with: From 1164aa4efc225129302e2a2450e907ad842d41e4 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 23 Apr 2023 19:35:54 -0700 Subject: [PATCH 306/459] Installing custom docker should no longer be necessary for buildx --- .multi_arch_docker | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/.multi_arch_docker b/.multi_arch_docker index a9f7401..1c5d32b 100755 --- a/.multi_arch_docker +++ b/.multi_arch_docker @@ -3,28 +3,10 @@ # binaries previously built and deployed to GitHub. function multi_arch_docker::install_docker_buildx() { - # Install up-to-date version of docker, with buildx support. - local -r docker_apt_repo='https://download.docker.com/linux/ubuntu' - curl -fsSL "${docker_apt_repo}/gpg" | sudo apt-key add - - local -r os="$(lsb_release -cs)" - sudo add-apt-repository "deb [arch=amd64] $docker_apt_repo $os stable" - sudo apt-get update - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce - - # Enable docker daemon experimental support (for 'pull --platform'). - local -r config='/etc/docker/daemon.json' - if [[ -e "$config" ]]; then - sudo sed -i -e 's/{/{ "experimental": true, /' "$config" - else - echo '{ "experimental": true }' | sudo tee "$config" - fi - sudo systemctl restart docker - # Install QEMU multi-architecture support for docker buildx. docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # Instantiate docker buildx builder with multi-architecture support. - export DOCKER_CLI_EXPERIMENTAL=enabled docker buildx create --name mybuilder docker buildx use mybuilder # Start up buildx and verify that all is OK. From 5fec3f9b34bc0ba3d2d02d4a7613cb3a62ed4c15 Mon Sep 17 00:00:00 2001 From: James Morris Date: Mon, 24 Apr 2023 22:08:22 -0400 Subject: [PATCH 307/459] Add fish to the badShells list --- src/ShellCheck/Parser.hs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 7a50967..9d7df00 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -3360,6 +3360,7 @@ readScriptFile sourced = do "awk", "csh", "expect", + "fish", "perl", "python", "ruby", From 46b678fca8f8aac035d04e676f77f1a92f6742f4 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 30 Apr 2023 14:37:37 -0700 Subject: [PATCH 308/459] Minor fixes to POSIX read without variable check --- src/ShellCheck/Checks/ShellSupport.hs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index eda6882..cf8acc9 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -188,6 +188,7 @@ prop_checkBashisms100 = verify checkBashisms "read -r" prop_checkBashisms101 = verify checkBashisms "read" prop_checkBashisms102 = verifyNot checkBashisms "read -r foo" prop_checkBashisms103 = verifyNot checkBashisms "read foo" +prop_checkBashisms104 = verifyNot checkBashisms "read ''" checkBashisms = ForShell [Sh, Dash] $ \t -> do params <- ask kludge params t @@ -288,13 +289,6 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do argString = concat $ oversimplify arg flagRegex = mkRegex "^-[eEsn]+$" - bashism t@(T_SimpleCommand _ _ (cmd:args)) - | t `isCommand` "read" && length (onlyNames args) == 0 = - warnMsg (getId cmd) 3061 "read without a variable is" - where - notFlag arg = head (concat $ oversimplify arg) /= '-' - onlyNames = filter (notFlag) - bashism t@(T_SimpleCommand _ _ (cmd:arg:_)) | getLiteralString cmd == Just "exec" && "-" `isPrefixOf` concat (oversimplify arg) = warnMsg (getId arg) 3038 "exec flags are" @@ -390,6 +384,9 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do let literal = onlyLiteralString format guard $ "%q" `isInfixOf` literal return $ warnMsg (getId format) 3050 "printf %q is" + + when (name == "read" && all isFlag rest) $ + warnMsg (getId cmd) 3061 "read without a variable is" where unsupportedCommands = [ "let", "caller", "builtin", "complete", "compgen", "declare", "dirs", "disown", From b3932dfa10804434fb8c15dc32e428c5a1c3bfa4 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Mon, 1 May 2023 00:02:53 -0400 Subject: [PATCH 309/459] Fix #2734: adjust bounds to compile on 9.6 The whole test suite passes for me, including prop_checkOverwrittenExitCode8, and I get the same set of findings with this build and shellcheck.net on tools/testing/selftests/net/icmp_redirect.sh. --- ShellCheck.cabal | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ShellCheck.cabal b/ShellCheck.cabal index 8afebe1..f09521f 100644 --- a/ShellCheck.cabal +++ b/ShellCheck.cabal @@ -46,7 +46,7 @@ library semigroups build-depends: -- The lower bounds are based on GHC 7.10.3 - -- The upper bounds are based on GHC 9.4.3 + -- The upper bounds are based on GHC 9.6.1 aeson >= 1.4.0 && < 2.2, array >= 0.5.1 && < 0.6, base >= 4.8.0.0 && < 5, @@ -54,13 +54,13 @@ library containers >= 0.5.6 && < 0.7, deepseq >= 1.4.1 && < 1.5, Diff >= 0.4.0 && < 0.5, - fgl >= 5.7.0 && < 5.8.1.0, + fgl (>= 5.7.0 && < 5.8.1.0) || (>= 5.8.1.1 && < 5.9), filepath >= 1.4.0 && < 1.5, - mtl >= 2.2.2 && < 2.3, + mtl >= 2.2.2 && < 2.4, parsec >= 3.1.14 && < 3.2, QuickCheck >= 2.14.2 && < 2.15, regex-tdfa >= 1.2.0 && < 1.4, - transformers >= 0.4.2 && < 0.6, + transformers >= 0.4.2 && < 0.7, -- getXdgDirectory from 1.2.3.0 directory >= 1.2.3 && < 1.4, From f03c437e2fe0d669d1e64dcedd305d5bd8ca0608 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Wed, 24 May 2023 16:38:53 -0400 Subject: [PATCH 310/459] Get rid of a dangerous partial function from checkSpacefulnessCfg' --- src/ShellCheck/Analytics.hs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 1f6d96d..ecc170d 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2134,7 +2134,8 @@ checkSpacefulnessCfg' dirtyPass params token@(T_DollarBraced id _ list) = addDoubleQuotesAround params token where - name = getBracedReference $ concat $ oversimplify list + bracedString = concat $ oversimplify list + name = getBracedReference bracedString parents = parentMap params needsQuoting = not (isArrayExpansion token) -- There's another warning for this @@ -2153,14 +2154,10 @@ checkSpacefulnessCfg' dirtyPass params token@(T_DollarBraced id _ list) = || CF.spaceStatus (CF.variableValue state) == CF.SpaceStatusClean isDefaultAssignment parents token = - let modifier = getBracedModifier $ bracedString token in + let modifier = getBracedModifier bracedString in any (`isPrefixOf` modifier) ["=", ":="] && isParamTo parents ":" token - -- Given a T_DollarBraced, return a simplified version of the string contents. - bracedString (T_DollarBraced _ _ l) = concat $ oversimplify l - bracedString _ = error $ pleaseReport "bracedString on non-variable" - checkSpacefulnessCfg' _ _ _ = return () From b625cc1accb3249322ca757b6709840f7582b072 Mon Sep 17 00:00:00 2001 From: Nicolas Theodarus Date: Sun, 28 May 2023 12:33:16 +0200 Subject: [PATCH 311/459] add dependabot.yml --- .github/dependabot.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..81bae9a --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,7 @@ +version: 2 + +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" From 01aee1a859913a02b529d1f25469693166d3fe7c Mon Sep 17 00:00:00 2001 From: Danny Faught Date: Fri, 28 Jul 2023 14:19:54 -0400 Subject: [PATCH 312/459] improve short description * The short description used to say that until commit aac7d76047a5b28d064b17a5d0fac022054d05a0 from 2014. It appears that it was changed by mistake in that commit to something less readable. * With the message "use -print0/-0" we were confused and introduced a bug in our code because we didn't understand what to do with the "-0". * SC2011 (source https://github.com/koalaman/shellcheck/blob/c9e27c24700cdc5b84cfca1f7a90fe07f542867c/src/ShellCheck/Analytics.hs#L591) uses that exact warning message, we copied it from there. Signed-off-by: Bruce Ricard --- src/ShellCheck/Analytics.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index ecc170d..3bb1ed0 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -562,7 +562,7 @@ checkPipePitfalls _ (T_Pipeline id _ commands) = do hasParameter "print0", hasParameter "printf" ]) $ warn (getId find) 2038 - "Use -print0/-0 or -exec + to allow for non-alphanumeric filenames." + "Use 'find .. -print0 | xargs -0 ..' or 'find .. -exec .. +' to allow non-alphanumeric filenames." for ["ps", "grep"] $ \(ps:grep:_) -> From 372c0b667e7b6f36a5f1a42a9802eb0246ee3e95 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 30 Jul 2023 13:47:00 -0700 Subject: [PATCH 313/459] SC2324: Warn when x+=1 appends. --- CHANGELOG.md | 1 + src/ShellCheck/ASTLib.hs | 9 +++++++++ src/ShellCheck/Analytics.hs | 38 +++++++++++++++++++++++++++++++++++ src/ShellCheck/CFGAnalysis.hs | 16 +++++++++++++++ 4 files changed, 64 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fc1c0ce..8f4426e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ## Git ### Added +- SC2324: Warn when x+=1 appends instead of increments. ### Fixed diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 56903ee..64fa762 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -886,6 +886,15 @@ isUnmodifiedParameterExpansion t = in getBracedReference str == str _ -> False +-- Return the referenced variable if (and only if) it's an unmodified parameter expansion. +getUnmodifiedParameterExpansion t = + case t of + T_DollarBraced _ _ list -> do + let str = concat $ oversimplify list + guard $ getBracedReference str == str + return str + _ -> Nothing + --- A list of the element and all its parents up to the root node. getPath tree t = t : case Map.lookup (getId t) tree of diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index ecc170d..dbad0f5 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -201,6 +201,7 @@ nodeChecks = [ ,checkOverwrittenExitCode ,checkUnnecessaryArithmeticExpansionIndex ,checkUnnecessaryParens + ,checkPlusEqualsNumber ] optionalChecks = map fst optionalTreeChecks @@ -5007,5 +5008,42 @@ checkUnnecessaryParens params t = ] +prop_checkPlusEqualsNumber1 = verify checkPlusEqualsNumber "x+=1" +prop_checkPlusEqualsNumber2 = verify checkPlusEqualsNumber "x+=42" +prop_checkPlusEqualsNumber3 = verifyNot checkPlusEqualsNumber "(( x += 1 ))" +prop_checkPlusEqualsNumber4 = verifyNot checkPlusEqualsNumber "declare -i x=0; x+=1" +prop_checkPlusEqualsNumber5 = verifyNot checkPlusEqualsNumber "x+='1'" +prop_checkPlusEqualsNumber6 = verifyNot checkPlusEqualsNumber "n=foo; x+=n" +prop_checkPlusEqualsNumber7 = verify checkPlusEqualsNumber "n=4; x+=n" +prop_checkPlusEqualsNumber8 = verify checkPlusEqualsNumber "n=4; x+=$n" +prop_checkPlusEqualsNumber9 = verifyNot checkPlusEqualsNumber "declare -ia var; var[x]+=1" +checkPlusEqualsNumber params t = + case t of + T_Assignment id Append var _ word -> sequence_ $ do + state <- CF.getIncomingState (cfgAnalysis params) id + guard $ isNumber state word + guard . not $ fromMaybe False $ CF.variableMayBeDeclaredInteger state var + return $ warn id 2324 "var+=1 will append, not increment. Use (( var += 1 )), declare -i var, or quote number to silence." + _ -> return () + + where + isNumber state word = + let + unquotedLiteral = getUnquotedLiteral word + isEmpty = unquotedLiteral == Just "" + isUnquotedNumber = not isEmpty && fromMaybe False (all isDigit <$> unquotedLiteral) + isNumericalVariableName = fromMaybe False $ do + str <- unquotedLiteral + CF.variableMayBeAssignedInteger state str + isNumericalVariableExpansion = + case word of + T_NormalWord _ [part] -> fromMaybe False $ do + str <- getUnmodifiedParameterExpansion part + CF.variableMayBeAssignedInteger state str + _ -> False + in + isUnquotedNumber || isNumericalVariableName || isNumericalVariableExpansion + + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) diff --git a/src/ShellCheck/CFGAnalysis.hs b/src/ShellCheck/CFGAnalysis.hs index 4e36cf5..3b4f957 100644 --- a/src/ShellCheck/CFGAnalysis.hs +++ b/src/ShellCheck/CFGAnalysis.hs @@ -59,6 +59,8 @@ module ShellCheck.CFGAnalysis ( ,getIncomingState ,getOutgoingState ,doesPostDominate + ,variableMayBeDeclaredInteger + ,variableMayBeAssignedInteger ,ShellCheck.CFGAnalysis.runTests -- STRIP ) where @@ -153,6 +155,20 @@ doesPostDominate analysis target base = fromMaybe False $ do (targetStart, _) <- M.lookup target $ tokenToRange analysis return $ targetStart `elem` (postDominators analysis ! baseEnd) +-- See if any execution path results in the variable containing a state +variableMayHaveState :: ProgramState -> String -> CFVariableProp -> Maybe Bool +variableMayHaveState state var property = do + value <- M.lookup var $ variablesInScope state + return $ any (S.member property) $ variableProperties value + +-- See if any execution path declares the variable an integer (declare -i). +variableMayBeDeclaredInteger state var = variableMayHaveState state var CFVPInteger + +-- See if any execution path suggests the variable may contain an integer value +variableMayBeAssignedInteger state var = do + value <- M.lookup var $ variablesInScope state + return $ (numericalStatus $ variableValue value) >= NumericalStatusMaybe + getDataForNode analysis node = M.lookup node $ nodeToData analysis -- The current state of data flow at a point in the program, potentially as a diff From 9490b9488627a06e0a4af1c11644b7936b8a2422 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 30 Jul 2023 16:52:40 -0700 Subject: [PATCH 314/459] Save and restore pending here docs when sourcing files (fixes #2803) --- CHANGELOG.md | 3 ++- src/ShellCheck/Checker.hs | 9 +++++++++ src/ShellCheck/Parser.hs | 10 ++++++++-- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f4426e..c6c9513 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,9 @@ ## Git ### Added -- SC2324: Warn when x+=1 appends instead of increments. +- SC2324: Warn when x+=1 appends instead of increments ### Fixed +- source statements with here docs now work correctly ### Changed diff --git a/src/ShellCheck/Checker.hs b/src/ShellCheck/Checker.hs index b56be68..c79f90f 100644 --- a/src/ShellCheck/Checker.hs +++ b/src/ShellCheck/Checker.hs @@ -508,5 +508,14 @@ prop_rcCanSuppressEarlyProblems2 = null result csScript = "!/bin/bash\necho 'hello world'" } +prop_sourceWithHereDocWorks = null result + where + result = checkWithIncludes [("bar", "true\n")] "source bar << eof\nlol\neof" + +prop_hereDocsAreParsedWithoutTrailingLinefeed = 1044 `elem` result + where + result = check "cat << eof" + + return [] runTests = $quickCheckAll diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 9d7df00..341a435 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -2283,8 +2283,13 @@ readSource t@(T_Redirecting _ _ (T_SimpleCommand cmdId _ (cmd:file':rest'))) = d subRead name script = withContext (ContextSource name) $ - inSeparateContext $ - subParse (initialPos name) (readScriptFile True) script + inSeparateContext $ do + oldState <- getState + setState $ oldState { pendingHereDocs = [] } + result <- subParse (initialPos name) (readScriptFile True) script + newState <- getState + setState $ newState { pendingHereDocs = pendingHereDocs oldState } + return result readSource t = return t @@ -3322,6 +3327,7 @@ readScriptFile sourced = do then do commands <- readCompoundListOrEmpty id <- endSpan start + readPendingHereDocs verifyEof let script = T_Annotation annotationId annotations $ T_Script id shebang commands From dd747b2a98c3214978a97b9ee0ec38e635b6e621 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 30 Jul 2023 19:18:27 -0700 Subject: [PATCH 315/459] SC2325/SC2326: Warn about ! ! foo and foo | ! bar (fixes #2810) --- CHANGELOG.md | 2 ++ src/ShellCheck/Checks/ShellSupport.hs | 26 ++++++++++++++++++++++++++ src/ShellCheck/Parser.hs | 22 +++++++++++++++------- 3 files changed, 43 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c6c9513..0338f7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ ## Git ### Added - SC2324: Warn when x+=1 appends instead of increments +- SC2325: Warn about multiple `!`s in dash/sh. +- SC2326: Warn about `foo | ! bar` in bash/dash/sh. ### Fixed - source statements with here docs now work correctly diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index cf8acc9..c7ece1a 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -60,6 +60,8 @@ checks = [ ,checkBraceExpansionVars ,checkMultiDimensionalArrays ,checkPS1Assignments + ,checkMultipleBangs + ,checkBangAfterPipe ] testChecker (ForShell _ t) = @@ -566,5 +568,29 @@ checkPS1Assignments = ForShell [Bash] f escapeRegex = mkRegex "\\\\x1[Bb]|\\\\e|\x1B|\\\\033" +prop_checkMultipleBangs1 = verify checkMultipleBangs "! ! true" +prop_checkMultipleBangs2 = verifyNot checkMultipleBangs "! true" +checkMultipleBangs = ForShell [Dash, Sh] f + where + f token = case token of + T_Banged id (T_Banged _ _) -> + err id 2325 "Multiple ! in front of pipelines are a bash/ksh extension. Use only 0 or 1." + _ -> return () + + +prop_checkBangAfterPipe1 = verify checkBangAfterPipe "true | ! true" +prop_checkBangAfterPipe2 = verifyNot checkBangAfterPipe "true | ( ! true )" +prop_checkBangAfterPipe3 = verifyNot checkBangAfterPipe "! ! true | true" +checkBangAfterPipe = ForShell [Dash, Sh, Bash] f + where + f token = case token of + T_Pipeline _ _ cmds -> mapM_ check cmds + _ -> return () + + check token = case token of + T_Banged id _ -> + err id 2326 "! is not allowed in the middle of pipelines. Use command group as in cmd | { ! cmd; } if necessary." + _ -> return () + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 341a435..ffc58e2 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -2296,14 +2296,18 @@ readSource t = return t prop_readPipeline = isOk readPipeline "! cat /etc/issue | grep -i ubuntu" prop_readPipeline2 = isWarning readPipeline "!cat /etc/issue | grep -i ubuntu" prop_readPipeline3 = isOk readPipeline "for f; do :; done|cat" +prop_readPipeline4 = isOk readPipeline "! ! true" +prop_readPipeline5 = isOk readPipeline "true | ! true" readPipeline = do unexpecting "keyword/token" readKeyword - do - (T_Bang id) <- g_Bang - pipe <- readPipeSequence - return $ T_Banged id pipe - <|> - readPipeSequence + readBanged readPipeSequence + +readBanged parser = do + pos <- getPosition + (T_Bang id) <- g_Bang + next <- readBanged parser + return $ T_Banged id next + <|> parser prop_readAndOr = isOk readAndOr "grep -i lol foo || exit 1" prop_readAndOr1 = isOk readAndOr "# shellcheck disable=1\nfoo" @@ -2359,7 +2363,7 @@ readTerm = do readPipeSequence = do start <- startSpan - (cmds, pipes) <- sepBy1WithSeparators readCommand + (cmds, pipes) <- sepBy1WithSeparators (readBanged readCommand) (readPipe `thenSkip` (spacing >> readLineBreak)) id <- endSpan start spacing @@ -2389,6 +2393,10 @@ readCommand = choice [ ] readCmdName = do + -- If the command name is `!` then + optional . lookAhead . try $ do + char '!' + whitespace -- Ignore alias suppression optional . try $ do char '\\' From 90d3172dfec30a7569f95b32479ae97af73b8b2e Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 13 Aug 2023 16:32:53 -0700 Subject: [PATCH 316/459] Add a newSystemInterface to go with the rest of the new* constructors --- shellcheck.hs | 2 +- src/ShellCheck/Interface.hs | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/shellcheck.hs b/shellcheck.hs index 4e8a155..6be9bb1 100644 --- a/shellcheck.hs +++ b/shellcheck.hs @@ -396,7 +396,7 @@ ioInterface options files = do inputs <- mapM normalize files cache <- newIORef emptyCache configCache <- newIORef ("", Nothing) - return SystemInterface { + return (newSystemInterface :: SystemInterface IO) { siReadFile = get cache inputs, siFindSource = findSourceFile inputs (sourcePaths options), siGetConfig = getConfig configCache diff --git a/src/ShellCheck/Interface.hs b/src/ShellCheck/Interface.hs index 7528559..077212f 100644 --- a/src/ShellCheck/Interface.hs +++ b/src/ShellCheck/Interface.hs @@ -39,11 +39,12 @@ module ShellCheck.Interface , ColorOption(ColorAuto, ColorAlways, ColorNever) , TokenComment(tcId, tcComment, tcFix) , emptyCheckResult - , newParseResult - , newAnalysisSpec , newAnalysisResult + , newAnalysisSpec , newFormatterOptions + , newParseResult , newPosition + , newSystemInterface , newTokenComment , mockedSystemInterface , mockRcFile @@ -135,6 +136,14 @@ newParseSpec = ParseSpec { psShellTypeOverride = Nothing } +newSystemInterface :: Monad m => SystemInterface m +newSystemInterface = + SystemInterface { + siReadFile = \_ _ -> return $ Left "Not implemented", + siFindSource = \_ _ _ name -> return name, + siGetConfig = \_ -> return Nothing + } + -- Parser input and output data ParseSpec = ParseSpec { psFilename :: String, @@ -311,7 +320,7 @@ data ColorOption = -- For testing mockedSystemInterface :: [(String, String)] -> SystemInterface Identity -mockedSystemInterface files = SystemInterface { +mockedSystemInterface files = (newSystemInterface :: SystemInterface Identity) { siReadFile = rf, siFindSource = fs, siGetConfig = const $ return Nothing From 410ec54617c86586cd6b5662ecc035898bd8aefa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 5 Sep 2023 08:21:55 +0000 Subject: [PATCH 317/459] Bump actions/checkout from 3 to 4 Bumps [actions/checkout](https://github.com/actions/checkout) from 3 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a435cf4..2ca94f2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,7 +15,7 @@ jobs: sudo apt-get install cabal-install - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 @@ -51,7 +51,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Download artifacts uses: actions/download-artifact@v3 @@ -74,7 +74,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Download artifacts uses: actions/download-artifact@v3 @@ -104,7 +104,7 @@ jobs: environment: Deploy steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Download artifacts uses: actions/download-artifact@v3 From c89ec2fd492d37722f2c93aaae500cc91a84e1c4 Mon Sep 17 00:00:00 2001 From: Max Ulidtko Date: Sun, 1 Oct 2023 19:57:19 +0200 Subject: [PATCH 318/459] Fix: do []-related bashism checks on test(1) calls too --- src/ShellCheck/Checks/ShellSupport.hs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index c7ece1a..1f012d2 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -19,6 +19,7 @@ -} {-# LANGUAGE TemplateHaskell #-} {-# LANGUAGE FlexibleContexts #-} +{-# LANGUAGE ViewPatterns #-} module ShellCheck.Checks.ShellSupport (checker , ShellCheck.Checks.ShellSupport.runTests) where import ShellCheck.AST @@ -91,6 +92,9 @@ prop_checkBashisms3 = verify checkBashisms "echo $((i++))" prop_checkBashisms4 = verify checkBashisms "rm !(*.hs)" prop_checkBashisms5 = verify checkBashisms "source file" prop_checkBashisms6 = verify checkBashisms "[ \"$a\" == 42 ]" +prop_checkBashisms6b = verify checkBashisms "test \"$a\" == 42" +prop_checkBashisms6c = verify checkBashisms "[ foo =~ bar ]" +prop_checkBashisms6d = verify checkBashisms "test foo =~ bar" prop_checkBashisms7 = verify checkBashisms "echo ${var[1]}" prop_checkBashisms8 = verify checkBashisms "echo ${!var[@]}" prop_checkBashisms9 = verify checkBashisms "echo ${!var*}" @@ -106,6 +110,7 @@ prop_checkBashisms18 = verify checkBashisms "foo &> /dev/null" prop_checkBashisms19 = verify checkBashisms "foo > file*.txt" prop_checkBashisms20 = verify checkBashisms "read -ra foo" prop_checkBashisms21 = verify checkBashisms "[ -a foo ]" +prop_checkBashisms21b = verify checkBashisms "test -a foo" prop_checkBashisms22 = verifyNot checkBashisms "[ foo -a bar ]" prop_checkBashisms23 = verify checkBashisms "trap mything ERR INT" prop_checkBashisms24 = verifyNot checkBashisms "trap mything INT TERM" @@ -203,6 +208,7 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do if isDash then err id code $ "In dash, " ++ s ++ " not supported." else warn id code $ "In POSIX sh, " ++ s ++ " undefined." + asStr = getLiteralString bashism (T_ProcSub id _ _) = warnMsg id 3001 "process substitution is" bashism (T_Extglob id _ _) = warnMsg id 3002 "extglob is" @@ -218,17 +224,31 @@ checkBashisms = ForShell [Sh, Dash] $ \t -> do bashism (TC_Binary id SingleBracket op _ _) | op `elem` [ "<", ">", "\\<", "\\>", "<=", ">=", "\\<=", "\\>="] = unless isDash $ warnMsg id 3012 $ "lexicographical " ++ op ++ " is" + bashism (T_SimpleCommand id _ [asStr -> Just "test", lhs, asStr -> Just op, rhs]) + | op `elem` [ "<", ">", "\\<", "\\>", "<=", ">=", "\\<=", "\\>="] = + unless isDash $ warnMsg id 3012 $ "lexicographical " ++ op ++ " is" bashism (TC_Binary id SingleBracket op _ _) | op `elem` [ "-ot", "-nt", "-ef" ] = unless isDash $ warnMsg id 3013 $ op ++ " is" + bashism (T_SimpleCommand id _ [asStr -> Just "test", lhs, asStr -> Just op, rhs]) + | op `elem` [ "-ot", "-nt", "-ef" ] = + unless isDash $ warnMsg id 3013 $ op ++ " is" bashism (TC_Binary id SingleBracket "==" _ _) = warnMsg id 3014 "== in place of = is" + bashism (T_SimpleCommand id _ [asStr -> Just "test", lhs, asStr -> Just "==", rhs]) = + warnMsg id 3014 "== in place of = is" bashism (TC_Binary id SingleBracket "=~" _ _) = warnMsg id 3015 "=~ regex matching is" + bashism (T_SimpleCommand id _ [asStr -> Just "test", lhs, asStr -> Just "=~", rhs]) = + warnMsg id 3015 "=~ regex matching is" bashism (TC_Unary id SingleBracket "-v" _) = warnMsg id 3016 "unary -v (in place of [ -n \"${var+x}\" ]) is" + bashism (T_SimpleCommand id _ [asStr -> Just "test", asStr -> Just "-v", _]) = + warnMsg id 3016 "unary -v (in place of [ -n \"${var+x}\" ]) is" bashism (TC_Unary id _ "-a" _) = warnMsg id 3017 "unary -a in place of -e is" + bashism (T_SimpleCommand id _ [asStr -> Just "test", asStr -> Just "-a", _]) = + warnMsg id 3017 "unary -a in place of -e is" bashism (TA_Unary id op _) | op `elem` [ "|++", "|--", "++|", "--|"] = warnMsg id 3018 $ filter (/= '|') op ++ " is" From 9605396bef40abdb830bc4c607c3736c007e3482 Mon Sep 17 00:00:00 2001 From: Max Ulidtko Date: Sun, 1 Oct 2023 21:23:25 +0200 Subject: [PATCH 319/459] Docs: describe fixes of PR #2837 in changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0338f7d..b9ce1fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,12 @@ - SC2324: Warn when x+=1 appends instead of increments - SC2325: Warn about multiple `!`s in dash/sh. - SC2326: Warn about `foo | ! bar` in bash/dash/sh. +- SC3012: Warn about lexicographic-compare bashism in test like in [ ] +- SC3013: Warn bashism `test _ -op/-nt/-ef _` like in [ ] +- SC3014: Warn bashism `test _ == _` like in [ ] +- SC3015: Warn bashism `test _ =~ _` like in [ ] +- SC3016: Warn bashism `test -v _` like in [ ] +- SC3017: Warn bashism `test -a _` like in [ ] ### Fixed - source statements with here docs now work correctly From 6a6d8e9fc488b3b20ab93da2915cccb42e208ddc Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 8 Oct 2023 18:52:05 -0700 Subject: [PATCH 320/459] Revert "Bump actions/checkout from 3 to 4" This reverts commit 410ec54617c86586cd6b5662ecc035898bd8aefa. --- .github/workflows/build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2ca94f2..a435cf4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,7 +15,7 @@ jobs: sudo apt-get install cabal-install - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v3 with: fetch-depth: 0 @@ -51,7 +51,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v3 - name: Download artifacts uses: actions/download-artifact@v3 @@ -74,7 +74,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v3 - name: Download artifacts uses: actions/download-artifact@v3 @@ -104,7 +104,7 @@ jobs: environment: Deploy steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v3 - name: Download artifacts uses: actions/download-artifact@v3 From 99a94421ab77397ae98e192a722ba7e61c103dec Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 8 Oct 2023 19:42:31 -0700 Subject: [PATCH 321/459] Manually install 'hub' dependency --- .github/workflows/build.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a435cf4..3e6fb27 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -103,6 +103,11 @@ jobs: runs-on: ubuntu-latest environment: Deploy steps: + - name: Install Dependencies + run: | + sudo apt-get update + sudo apt-get install hub + - name: Checkout repository uses: actions/checkout@v3 From dc2f388310c48eefe4613edf71a703f07d02ffa7 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sat, 14 Oct 2023 18:12:51 -0400 Subject: [PATCH 322/459] Adjust bounds to compile on 9.8 You'll need --allow-newer=fgl:deepseq for it to work too, until haskell/fgl#111 gets merged. --- ShellCheck.cabal | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ShellCheck.cabal b/ShellCheck.cabal index f09521f..76516db 100644 --- a/ShellCheck.cabal +++ b/ShellCheck.cabal @@ -46,13 +46,13 @@ library semigroups build-depends: -- The lower bounds are based on GHC 7.10.3 - -- The upper bounds are based on GHC 9.6.1 - aeson >= 1.4.0 && < 2.2, + -- The upper bounds are based on GHC 9.8.1 + aeson >= 1.4.0 && < 2.3, array >= 0.5.1 && < 0.6, base >= 4.8.0.0 && < 5, - bytestring >= 0.10.6 && < 0.12, - containers >= 0.5.6 && < 0.7, - deepseq >= 1.4.1 && < 1.5, + bytestring >= 0.10.6 && < 0.13, + containers >= 0.5.6 && < 0.8, + deepseq >= 1.4.1 && < 1.6, Diff >= 0.4.0 && < 0.5, fgl (>= 5.7.0 && < 5.8.1.0) || (>= 5.8.1.1 && < 5.9), filepath >= 1.4.0 && < 1.5, From 8b3c37aa36bcb74b09d71eba4ce0ce0141b8bd4f Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Mon, 16 Oct 2023 00:06:53 -0400 Subject: [PATCH 323/459] Use find instead of listToMaybe and filter --- src/ShellCheck/Analytics.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 7e8b510..3cebb24 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -3320,7 +3320,7 @@ checkReturnAgainstZero params token = isFirstCommandInFunction = fromMaybe False $ do let path = getPath (parentMap params) token - func <- listToMaybe $ filter isFunction path + func <- find isFunction path cmd <- getClosestCommand (parentMap params) token return $ getId cmd == getId (getFirstCommandInFunction func) From 4fd0615501b6909c8ca61ef080bb07b8e8de1301 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Mon, 16 Oct 2023 00:55:04 -0400 Subject: [PATCH 324/459] Stop using head in isLeadingNumberVar --- src/ShellCheck/Analytics.hs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 3cebb24..5f77f77 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -4378,9 +4378,8 @@ checkEqualsInCommand params originalToken = return $ isVariableName str isLeadingNumberVar s = - let lead = takeWhile (/= '=') s - in not (null lead) && isDigit (head lead) - && all isVariableChar lead && not (all isDigit lead) + case takeWhile (/= '=') s of + lead@(x:_) -> isDigit x && all isVariableChar lead && not (all isDigit lead) msg cmd leading (T_Literal litId s) = do -- There are many different cases, and the order of the branches matter. From 2a95bc6be3c9fe411e1ddb96e78b467ef2347bdb Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Mon, 16 Oct 2023 20:00:31 -0400 Subject: [PATCH 325/459] Switch to getLiteralStringDef to avoid an unnecessary fromJust --- src/ShellCheck/CFG.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/CFG.hs b/src/ShellCheck/CFG.hs index f882adc..2fe11e7 100644 --- a/src/ShellCheck/CFG.hs +++ b/src/ShellCheck/CFG.hs @@ -1058,7 +1058,7 @@ handleCommand cmd vars args literalCmd = do let id = getId t pre = [t] - literal = fromJust $ getLiteralStringExt (const $ Just "\0") t + literal = getLiteralStringDef "\0" t isKnown = '\0' `notElem` literal match = fmap head $ variableAssignRegex `matchRegex` literal name = fromMaybe literal match From 1aeab287e62ab562142fdb8a42e3f486f93216dd Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Fri, 3 Nov 2023 01:33:49 -0400 Subject: [PATCH 326/459] Add nil case that went missing in 4fd0615 --- src/ShellCheck/Analytics.hs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 5f77f77..dadf512 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -4380,6 +4380,7 @@ checkEqualsInCommand params originalToken = isLeadingNumberVar s = case takeWhile (/= '=') s of lead@(x:_) -> isDigit x && all isVariableChar lead && not (all isDigit lead) + [] -> False msg cmd leading (T_Literal litId s) = do -- There are many different cases, and the order of the branches matter. From be8e4b2b8aac4177ef31092397f1ea2cad8e66ad Mon Sep 17 00:00:00 2001 From: Grische Date: Sat, 25 Nov 2023 12:44:46 +0100 Subject: [PATCH 327/459] add basic busybox sh support --- CHANGELOG.md | 1 + src/ShellCheck/ASTLib.hs | 7 ++++--- src/ShellCheck/Analytics.hs | 4 ++-- src/ShellCheck/AnalyzerLib.hs | 4 ++-- src/ShellCheck/Data.hs | 2 ++ src/ShellCheck/Interface.hs | 5 ++--- src/ShellCheck/Parser.hs | 5 +++-- 7 files changed, 16 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b9ce1fb..897aa27 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ - SC3015: Warn bashism `test _ =~ _` like in [ ] - SC3016: Warn bashism `test -v _` like in [ ] - SC3017: Warn bashism `test -a _` like in [ ] +- Added support for busybox sh ### Fixed - source statements with here docs now work correctly diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 64fa762..cf55498 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -758,8 +758,8 @@ prop_executableFromShebang6 = executableFromShebang "/usr/bin/env --split-string prop_executableFromShebang7 = executableFromShebang "/usr/bin/env --split-string bash -x" == "bash" prop_executableFromShebang8 = executableFromShebang "/usr/bin/env --split-string foo=bar bash -x" == "bash" prop_executableFromShebang9 = executableFromShebang "/usr/bin/env foo=bar dash" == "dash" -prop_executableFromShebang10 = executableFromShebang "/bin/busybox sh" == "ash" -prop_executableFromShebang11 = executableFromShebang "/bin/busybox ash" == "ash" +prop_executableFromShebang10 = executableFromShebang "/bin/busybox sh" == "busybox sh" +prop_executableFromShebang11 = executableFromShebang "/bin/busybox ash" == "busybox ash" -- Get the shell executable from a string like '/usr/bin/env bash' executableFromShebang :: String -> String @@ -776,7 +776,8 @@ executableFromShebang = shellFor [x] -> basename x (first:second:args) | basename first == "busybox" -> case basename second of - "sh" -> "ash" -- busybox sh is ash + "sh" -> "busybox sh" + "ash" -> "busybox ash" x -> x (first:args) | basename first == "env" -> fromEnvArgs args diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index dadf512..5df2f35 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -646,10 +646,10 @@ prop_checkShebang9 = verifyNotTree checkShebang "# shellcheck shell=sh\ntrue" prop_checkShebang10 = verifyNotTree checkShebang "#!foo\n# shellcheck shell=sh ignore=SC2239\ntrue" prop_checkShebang11 = verifyTree checkShebang "#!/bin/sh/\ntrue" prop_checkShebang12 = verifyTree checkShebang "#!/bin/sh/ -xe\ntrue" -prop_checkShebang13 = verifyTree checkShebang "#!/bin/busybox sh" +prop_checkShebang13 = verifyNotTree checkShebang "#!/bin/busybox sh" prop_checkShebang14 = verifyNotTree checkShebang "#!/bin/busybox sh\n# shellcheck shell=sh\n" prop_checkShebang15 = verifyNotTree checkShebang "#!/bin/busybox sh\n# shellcheck shell=dash\n" -prop_checkShebang16 = verifyTree checkShebang "#!/bin/busybox ash" +prop_checkShebang16 = verifyNotTree checkShebang "#!/bin/busybox ash" prop_checkShebang17 = verifyNotTree checkShebang "#!/bin/busybox ash\n# shellcheck shell=dash\n" prop_checkShebang18 = verifyNotTree checkShebang "#!/bin/busybox ash\n# shellcheck shell=sh\n" checkShebang params (T_Annotation _ list t) = diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index ca928fd..d864e32 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -284,8 +284,8 @@ prop_determineShell7 = determineShellTest "#! /bin/ash" == Dash prop_determineShell8 = determineShellTest' (Just Ksh) "#!/bin/sh" == Sh prop_determineShell9 = determineShellTest "#!/bin/env -S dash -x" == Dash prop_determineShell10 = determineShellTest "#!/bin/env --split-string= dash -x" == Dash -prop_determineShell11 = determineShellTest "#!/bin/busybox sh" == Dash -- busybox sh is a specific shell, not posix sh -prop_determineShell12 = determineShellTest "#!/bin/busybox ash" == Dash +prop_determineShell11 = determineShellTest "#!/bin/busybox sh" == BusyboxSh -- busybox sh is a specific shell, not posix sh +prop_determineShell12 = determineShellTest "#!/bin/busybox ash" == BusyboxSh determineShellTest = determineShellTest' Nothing determineShellTest' fallbackShell = determineShell fallbackShell . fromJust . prRoot . pScript diff --git a/src/ShellCheck/Data.hs b/src/ShellCheck/Data.hs index 550ff87..6a87123 100644 --- a/src/ShellCheck/Data.hs +++ b/src/ShellCheck/Data.hs @@ -156,6 +156,8 @@ shellForExecutable name = "sh" -> return Sh "bash" -> return Bash "bats" -> return Bash + "busybox sh" -> return BusyboxSh + "busybox ash" -> return BusyboxSh "dash" -> return Dash "ash" -> return Dash -- There's also a warning for this. "ksh" -> return Ksh diff --git a/src/ShellCheck/Interface.hs b/src/ShellCheck/Interface.hs index 077212f..c574cee 100644 --- a/src/ShellCheck/Interface.hs +++ b/src/ShellCheck/Interface.hs @@ -28,7 +28,7 @@ module ShellCheck.Interface , AnalysisSpec(asScript, asShellType, asFallbackShell, asExecutionMode, asCheckSourced, asTokenPositions, asOptionalChecks) , AnalysisResult(arComments) , FormatterOptions(foColorOption, foWikiLinkCount) - , Shell(Ksh, Sh, Bash, Dash) + , Shell(Ksh, Sh, Bash, Dash, BusyboxSh) , ExecutionMode(Executed, Sourced) , ErrorMessage , Code @@ -221,7 +221,7 @@ newCheckDescription = CheckDescription { } -- Supporting data types -data Shell = Ksh | Sh | Bash | Dash deriving (Show, Eq) +data Shell = Ksh | Sh | Bash | Dash | BusyboxSh deriving (Show, Eq) data ExecutionMode = Executed | Sourced deriving (Show, Eq) type ErrorMessage = String @@ -335,4 +335,3 @@ mockedSystemInterface files = (newSystemInterface :: SystemInterface Identity) { mockRcFile rcfile mock = mock { siGetConfig = const . return $ Just (".shellcheckrc", rcfile) } - diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index ffc58e2..abd4d94 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -3349,8 +3349,8 @@ readScriptFile sourced = do verifyShebang pos s = do case isValidShell s of Just True -> return () - Just False -> parseProblemAt pos ErrorC 1071 "ShellCheck only supports sh/bash/dash/ksh scripts. Sorry!" - Nothing -> parseProblemAt pos ErrorC 1008 "This shebang was unrecognized. ShellCheck only supports sh/bash/dash/ksh. Add a 'shell' directive to specify." + Just False -> parseProblemAt pos ErrorC 1071 "ShellCheck only supports sh/bash/dash/ksh/'busybox sh' scripts. Sorry!" + Nothing -> parseProblemAt pos ErrorC 1008 "This shebang was unrecognized. ShellCheck only supports sh/bash/dash/ksh/'busybox sh'. Add a 'shell' directive to specify." isValidShell s = let good = null s || any (`isPrefixOf` s) goodShells @@ -3366,6 +3366,7 @@ readScriptFile sourced = do "sh", "ash", "dash", + "busybox sh", "bash", "bats", "ksh" From 1e1045e73e0c66a947390785e95d924203e837fc Mon Sep 17 00:00:00 2001 From: Grische Date: Sat, 25 Nov 2023 12:52:32 +0100 Subject: [PATCH 328/459] make busybox sh Dash-like --- src/ShellCheck/Analytics.hs | 6 +++++- src/ShellCheck/AnalyzerLib.hs | 4 ++++ src/ShellCheck/Checks/Commands.hs | 4 ++-- src/ShellCheck/Checks/ShellSupport.hs | 12 +++++++----- 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 5df2f35..108682a 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1204,6 +1204,7 @@ checkNumberComparisons params (TC_Binary id typ op lhs rhs) = do case shellType params of Sh -> return () -- These are unsupported and will be caught by bashism checks. Dash -> err id 2073 $ "Escape \\" ++ op ++ " to prevent it redirecting." + BusyboxSh -> err id 2073 $ "Escape \\" ++ op ++ " to prevent it redirecting." _ -> err id 2073 $ "Escape \\" ++ op ++ " to prevent it redirecting (or switch to [[ .. ]])." when (op `elem` arithmeticBinaryTestOps) $ do @@ -2782,6 +2783,7 @@ checkFunctionDeclarations params when (hasKeyword && hasParens) $ err id 2111 "ksh does not allow 'function' keyword and '()' at the same time." Dash -> forSh + BusyboxSh -> forSh Sh -> forSh where @@ -4044,7 +4046,8 @@ prop_checkModifiedArithmeticInRedirection3 = verifyNot checkModifiedArithmeticIn prop_checkModifiedArithmeticInRedirection4 = verify checkModifiedArithmeticInRedirection "cat <<< $((i++))" prop_checkModifiedArithmeticInRedirection5 = verify checkModifiedArithmeticInRedirection "cat << foo\n$((i++))\nfoo\n" prop_checkModifiedArithmeticInRedirection6 = verifyNot checkModifiedArithmeticInRedirection "#!/bin/dash\nls > $((i=i+1))" -checkModifiedArithmeticInRedirection params t = unless (shellType params == Dash) $ +prop_checkModifiedArithmeticInRedirection7 = verifyNot checkModifiedArithmeticInRedirection "#!/bin/busybox sh\ncat << foo\n$((i++))\nfoo\n" +checkModifiedArithmeticInRedirection params t = unless (shellType params == Dash || shellType params == BusyboxSh) $ case t of T_Redirecting _ redirs (T_SimpleCommand _ _ (_:_)) -> mapM_ checkRedirs redirs _ -> return () @@ -4356,6 +4359,7 @@ checkEqualsInCommand params originalToken = Bash -> errWithFix id 2277 "Use BASH_ARGV0 to assign to $0 in bash (or use [ ] to compare)." bashfix Ksh -> err id 2278 "$0 can't be assigned in Ksh (but it does reflect the current function)." Dash -> err id 2279 "$0 can't be assigned in Dash. This becomes a command name." + BusyboxSh -> err id 2279 "$0 can't be assigned in Busybox Ash. This becomes a command name." _ -> err id 2280 "$0 can't be assigned this way, and there is no portable alternative." leadingNumberMsg id = err id 2282 "Variable names can't start with numbers, so this is interpreted as a command." diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index d864e32..4990822 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -206,18 +206,21 @@ makeParameters spec = params case shellType params of Bash -> isOptionSet "lastpipe" root Dash -> False + BusyboxSh -> False Sh -> False Ksh -> True, hasInheritErrexit = case shellType params of Bash -> isOptionSet "inherit_errexit" root Dash -> True + BusyboxSh -> True Sh -> True Ksh -> False, hasPipefail = case shellType params of Bash -> isOptionSet "pipefail" root Dash -> True + BusyboxSh -> isOptionSet "pipefail" root Sh -> True Ksh -> isOptionSet "pipefail" root, shellTypeSpecified = isJust (asShellType spec) || isJust (asFallbackShell spec), @@ -899,6 +902,7 @@ isBashLike params = Bash -> True Ksh -> True Dash -> False + BusyboxSh -> False Sh -> False isTrueAssignmentSource c = diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 691836f..8be60a7 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -930,7 +930,7 @@ prop_checkTimedCommand2 = verify checkTimedCommand "#!/bin/dash\ntime ( foo; bar prop_checkTimedCommand3 = verifyNot checkTimedCommand "#!/bin/sh\ntime sleep 1" checkTimedCommand = CommandCheck (Exactly "time") f where f (T_SimpleCommand _ _ (c:args@(_:_))) = - whenShell [Sh, Dash] $ do + whenShell [Sh, Dash, BusyboxSh] $ do let cmd = last args -- "time" is parsed with a command as argument when (isPiped cmd) $ warn (getId c) 2176 "'time' is undefined for pipelines. time single stage or bash -c instead." @@ -954,7 +954,7 @@ checkTimedCommand = CommandCheck (Exactly "time") f where prop_checkLocalScope1 = verify checkLocalScope "local foo=3" prop_checkLocalScope2 = verifyNot checkLocalScope "f() { local foo=3; }" checkLocalScope = CommandCheck (Exactly "local") $ \t -> - whenShell [Bash, Dash] $ do -- Ksh allows it, Sh doesn't support local + whenShell [Bash, Dash, BusyboxSh] $ do -- Ksh allows it, Sh doesn't support local path <- getPathM t unless (any isFunctionLike path) $ err (getId $ getCommandTokenOrThis t) 2168 "'local' is only valid in functions." diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 1f012d2..7112b92 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -76,7 +76,7 @@ verifyNot c s = producesComments (testChecker c) s == Just False prop_checkForDecimals1 = verify checkForDecimals "((3.14*c))" prop_checkForDecimals2 = verify checkForDecimals "foo[1.2]=bar" prop_checkForDecimals3 = verifyNot checkForDecimals "declare -A foo; foo[1.2]=bar" -checkForDecimals = ForShell [Sh, Dash, Bash] f +checkForDecimals = ForShell [Sh, Dash, BusyboxSh, Bash] f where f t@(TA_Expansion id _) = sequence_ $ do str <- getLiteralString t @@ -196,14 +196,16 @@ prop_checkBashisms101 = verify checkBashisms "read" prop_checkBashisms102 = verifyNot checkBashisms "read -r foo" prop_checkBashisms103 = verifyNot checkBashisms "read foo" prop_checkBashisms104 = verifyNot checkBashisms "read ''" -checkBashisms = ForShell [Sh, Dash] $ \t -> do +prop_checkBashisms105 = verifyNot checkBashisms "#!/bin/busybox sh\nset -o pipefail" +checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do params <- ask kludge params t where -- This code was copy-pasted from Analytics where params was a variable kludge params = bashism where - isDash = shellType params == Dash + isBusyboxSh = shellType params == BusyboxSh + isDash = shellType params == Dash || isBusyboxSh warnMsg id code s = if isDash then err id code $ "In dash, " ++ s ++ " not supported." @@ -590,7 +592,7 @@ checkPS1Assignments = ForShell [Bash] f prop_checkMultipleBangs1 = verify checkMultipleBangs "! ! true" prop_checkMultipleBangs2 = verifyNot checkMultipleBangs "! true" -checkMultipleBangs = ForShell [Dash, Sh] f +checkMultipleBangs = ForShell [Dash, BusyboxSh, Sh] f where f token = case token of T_Banged id (T_Banged _ _) -> @@ -601,7 +603,7 @@ checkMultipleBangs = ForShell [Dash, Sh] f prop_checkBangAfterPipe1 = verify checkBangAfterPipe "true | ! true" prop_checkBangAfterPipe2 = verifyNot checkBangAfterPipe "true | ( ! true )" prop_checkBangAfterPipe3 = verifyNot checkBangAfterPipe "! ! true | true" -checkBangAfterPipe = ForShell [Dash, Sh, Bash] f +checkBangAfterPipe = ForShell [Dash, BusyboxSh, Sh, Bash] f where f token = case token of T_Pipeline _ _ cmds -> mapM_ check cmds From 00ffd2db33724c31d8857d278379f8140a3f5fce Mon Sep 17 00:00:00 2001 From: Grische Date: Sat, 25 Nov 2023 13:50:23 +0100 Subject: [PATCH 329/459] silence SC3010 for busybox sh --- src/ShellCheck/Checks/ShellSupport.hs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 7112b92..9ade1c1 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -197,6 +197,7 @@ prop_checkBashisms102 = verifyNot checkBashisms "read -r foo" prop_checkBashisms103 = verifyNot checkBashisms "read foo" prop_checkBashisms104 = verifyNot checkBashisms "read ''" prop_checkBashisms105 = verifyNot checkBashisms "#!/bin/busybox sh\nset -o pipefail" +prop_checkBashisms106 = verifyNot checkBashisms "#!/bin/busybox sh\nx=x\n[[ \"$x\" = \"$x\" ]]" checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do params <- ask kludge params t @@ -221,7 +222,8 @@ checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do bashism (T_DollarBracket id _) = warnMsg id 3007 "$[..] in place of $((..)) is" bashism (T_SelectIn id _ _ _) = warnMsg id 3008 "select loops are" bashism (T_BraceExpansion id _) = warnMsg id 3009 "brace expansion is" - bashism (T_Condition id DoubleBracket _) = warnMsg id 3010 "[[ ]] is" + bashism (T_Condition id DoubleBracket _) = + unless isBusyboxSh $ warnMsg id 3010 "[[ ]] is" bashism (T_HereString id _) = warnMsg id 3011 "here-strings are" bashism (TC_Binary id SingleBracket op _ _) | op `elem` [ "<", ">", "\\<", "\\>", "<=", ">=", "\\<=", "\\>="] = From 903421fb5dff31993c07b8b25eb0d46c388cfba0 Mon Sep 17 00:00:00 2001 From: Grische Date: Sat, 25 Nov 2023 13:53:13 +0100 Subject: [PATCH 330/459] silence SC3014 for busybox sh --- src/ShellCheck/Checks/ShellSupport.hs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 9ade1c1..e1a09dd 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -198,6 +198,7 @@ prop_checkBashisms103 = verifyNot checkBashisms "read foo" prop_checkBashisms104 = verifyNot checkBashisms "read ''" prop_checkBashisms105 = verifyNot checkBashisms "#!/bin/busybox sh\nset -o pipefail" prop_checkBashisms106 = verifyNot checkBashisms "#!/bin/busybox sh\nx=x\n[[ \"$x\" = \"$x\" ]]" +prop_checkBashisms107 = verifyNot checkBashisms "#!/bin/busybox sh\nx=x\n[ \"$x\" == \"$x\" ]" checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do params <- ask kludge params t @@ -238,9 +239,9 @@ checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do | op `elem` [ "-ot", "-nt", "-ef" ] = unless isDash $ warnMsg id 3013 $ op ++ " is" bashism (TC_Binary id SingleBracket "==" _ _) = - warnMsg id 3014 "== in place of = is" + unless isBusyboxSh $ warnMsg id 3014 "== in place of = is" bashism (T_SimpleCommand id _ [asStr -> Just "test", lhs, asStr -> Just "==", rhs]) = - warnMsg id 3014 "== in place of = is" + unless isBusyboxSh $ warnMsg id 3014 "== in place of = is" bashism (TC_Binary id SingleBracket "=~" _ _) = warnMsg id 3015 "=~ regex matching is" bashism (T_SimpleCommand id _ [asStr -> Just "test", lhs, asStr -> Just "=~", rhs]) = From ac63dc33c9b9b3576bc6e3f44049a9e6a53dfaea Mon Sep 17 00:00:00 2001 From: Grische Date: Sat, 25 Nov 2023 13:55:07 +0100 Subject: [PATCH 331/459] silence SC3020 for busybox sh --- src/ShellCheck/Checks/ShellSupport.hs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index e1a09dd..4409751 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -199,6 +199,7 @@ prop_checkBashisms104 = verifyNot checkBashisms "read ''" prop_checkBashisms105 = verifyNot checkBashisms "#!/bin/busybox sh\nset -o pipefail" prop_checkBashisms106 = verifyNot checkBashisms "#!/bin/busybox sh\nx=x\n[[ \"$x\" = \"$x\" ]]" prop_checkBashisms107 = verifyNot checkBashisms "#!/bin/busybox sh\nx=x\n[ \"$x\" == \"$x\" ]" +prop_checkBashisms108 = verifyNot checkBashisms "#!/bin/busybox sh\necho magic &> /dev/null" checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do params <- ask kludge params t @@ -258,7 +259,8 @@ checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do | op `elem` [ "|++", "|--", "++|", "--|"] = warnMsg id 3018 $ filter (/= '|') op ++ " is" bashism (TA_Binary id "**" _ _) = warnMsg id 3019 "exponentials are" - bashism (T_FdRedirect id "&" (T_IoFile _ (T_Greater _) _)) = warnMsg id 3020 "&> is" + bashism (T_FdRedirect id "&" (T_IoFile _ (T_Greater _) _)) = + unless isBusyboxSh $ warnMsg id 3020 "&> is" bashism (T_FdRedirect id "" (T_IoFile _ (T_GREATAND _) file)) = unless (all isDigit $ onlyLiteralString file) $ warnMsg id 3021 ">& filename (as opposed to >& fd) is" bashism (T_FdRedirect id ('{':_) _) = warnMsg id 3022 "named file descriptors are" From a3b8be82fe2b5d41837463bf81b1a92bb84e3835 Mon Sep 17 00:00:00 2001 From: Grische Date: Sat, 25 Nov 2023 13:58:49 +0100 Subject: [PATCH 332/459] silence SC3048 for busybox sh --- src/ShellCheck/Checks/ShellSupport.hs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 4409751..6ebdd70 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -200,6 +200,7 @@ prop_checkBashisms105 = verifyNot checkBashisms "#!/bin/busybox sh\nset -o pipef prop_checkBashisms106 = verifyNot checkBashisms "#!/bin/busybox sh\nx=x\n[[ \"$x\" = \"$x\" ]]" prop_checkBashisms107 = verifyNot checkBashisms "#!/bin/busybox sh\nx=x\n[ \"$x\" == \"$x\" ]" prop_checkBashisms108 = verifyNot checkBashisms "#!/bin/busybox sh\necho magic &> /dev/null" +prop_checkBashisms109 = verifyNot checkBashisms "#!/bin/busybox sh\ntrap stop EXIT SIGTERM" checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do params <- ask kludge params t @@ -399,7 +400,7 @@ checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do return $ do when (upper `elem` ["ERR", "DEBUG", "RETURN"]) $ warnMsg (getId token) 3047 $ "trapping " ++ str ++ " is" - when ("SIG" `isPrefixOf` upper) $ + when (not isBusyboxSh && "SIG" `isPrefixOf` upper) $ warnMsg (getId token) 3048 "prefixing signal names with 'SIG' is" when (not isDash && upper /= str) $ From ca255fe3263e53c7cb21618eac7d5e52e5c0a665 Mon Sep 17 00:00:00 2001 From: Grische Date: Sat, 25 Nov 2023 14:04:11 +0100 Subject: [PATCH 333/459] silence SC3046 and SC3051 for busybox sh --- src/ShellCheck/Checks/ShellSupport.hs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 6ebdd70..f4c93fc 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -201,6 +201,7 @@ prop_checkBashisms106 = verifyNot checkBashisms "#!/bin/busybox sh\nx=x\n[[ \"$x prop_checkBashisms107 = verifyNot checkBashisms "#!/bin/busybox sh\nx=x\n[ \"$x\" == \"$x\" ]" prop_checkBashisms108 = verifyNot checkBashisms "#!/bin/busybox sh\necho magic &> /dev/null" prop_checkBashisms109 = verifyNot checkBashisms "#!/bin/busybox sh\ntrap stop EXIT SIGTERM" +prop_checkBashisms110 = verifyNot checkBashisms "#!/bin/busybox sh\nsource /dev/null" checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do params <- ask kludge params t @@ -391,7 +392,8 @@ checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do (\x -> (not . null . snd $ x) && snd x `notElem` allowed) flags return . warnMsg (getId word) 3045 $ name ++ " -" ++ flag ++ " is" - when (name == "source") $ warnMsg id 3046 "'source' in place of '.' is" + when (name == "source" && not isBusyboxSh) $ + warnMsg id 3046 "'source' in place of '.' is" when (name == "trap") $ let check token = sequence_ $ do @@ -440,7 +442,9 @@ checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do ("wait", Just []) ] bashism t@(T_SourceCommand id src _) - | getCommandName src == Just "source" = warnMsg id 3051 "'source' in place of '.' is" + | getCommandName src == Just "source" = + unless isBusyboxSh $ + warnMsg id 3051 "'source' in place of '.' is" bashism (TA_Expansion _ (T_Literal id str : _)) | str `matches` radix = warnMsg id 3052 "arithmetic base conversion is" where From fdcce458c189848b76d3779e35581cd012af201d Mon Sep 17 00:00:00 2001 From: Grische Date: Sat, 25 Nov 2023 15:10:44 +0100 Subject: [PATCH 334/459] silence some shell expansions for busybox sh --- src/ShellCheck/Checks/ShellSupport.hs | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index f4c93fc..e652cb5 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -202,6 +202,15 @@ prop_checkBashisms107 = verifyNot checkBashisms "#!/bin/busybox sh\nx=x\n[ \"$x\ prop_checkBashisms108 = verifyNot checkBashisms "#!/bin/busybox sh\necho magic &> /dev/null" prop_checkBashisms109 = verifyNot checkBashisms "#!/bin/busybox sh\ntrap stop EXIT SIGTERM" prop_checkBashisms110 = verifyNot checkBashisms "#!/bin/busybox sh\nsource /dev/null" +prop_checkBashisms111 = verify checkBashisms "#!/bin/dash\nx='test'\n${x:0:3}" -- SC3057 +prop_checkBashisms112 = verifyNot checkBashisms "#!/bin/busybox sh\nx='test'\n${x:0:3}" -- SC3057 +prop_checkBashisms113 = verify checkBashisms "#!/bin/dash\nx='test'\n${x/st/xt}" -- SC3060 +prop_checkBashisms114 = verifyNot checkBashisms "#!/bin/busybox sh\nx='test'\n${x/st/xt}" -- SC3060 +prop_checkBashisms115 = verify checkBashisms "#!/bin/busybox sh\nx='test'\n${!x}" -- SC3053 +prop_checkBashisms116 = verify checkBashisms "#!/bin/busybox sh\nx='test'\n${x[1]}" -- SC3054 +prop_checkBashisms117 = verify checkBashisms "#!/bin/busybox sh\nx='test'\n${!x[@]}" -- SC3055 +prop_checkBashisms118 = verify checkBashisms "#!/bin/busybox sh\nxyz=1\n${!x*}" -- SC3056 +prop_checkBashisms119 = verify checkBashisms "#!/bin/busybox sh\nx='test'\n${x^^[t]}" -- SC3059 checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do params <- ask kludge params t @@ -282,7 +291,8 @@ checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do warnMsg id 3028 $ str ++ " is" bashism t@(T_DollarBraced id _ token) = do - mapM_ check expansion + unless isBusyboxSh $ mapM_ check simpleExpansions + mapM_ check advancedExpansions when (isBashVariable var) $ warnMsg id 3028 $ var ++ " is" where @@ -452,14 +462,16 @@ checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do bashism _ = return () varChars="_0-9a-zA-Z" - expansion = let re = mkRegex in [ + advancedExpansions = let re = mkRegex in [ (re $ "^![" ++ varChars ++ "]", 3053, "indirect expansion is"), (re $ "^[" ++ varChars ++ "]+\\[.*\\]$", 3054, "array references are"), (re $ "^![" ++ varChars ++ "]+\\[[*@]]$", 3055, "array key expansion is"), (re $ "^![" ++ varChars ++ "]+[*@]$", 3056, "name matching prefixes are"), + (re $ "^[" ++ varChars ++ "*@]+(\\[.*\\])?[,^]", 3059, "case modification is") + ] + simpleExpansions = let re = mkRegex in [ (re $ "^[" ++ varChars ++ "*@]+:[^-=?+]", 3057, "string indexing is"), (re $ "^([*@][%#]|#[@*])", 3058, "string operations on $@/$* are"), - (re $ "^[" ++ varChars ++ "*@]+(\\[.*\\])?[,^]", 3059, "case modification is"), (re $ "^[" ++ varChars ++ "*@]+(\\[.*\\])?/", 3060, "string replacement is") ] bashVars = [ From b6d4952e2e0602c894f6d9f28e7004c205cbcae7 Mon Sep 17 00:00:00 2001 From: Andreas Abel Date: Wed, 6 Dec 2023 18:41:53 +0100 Subject: [PATCH 335/459] Testsuite: report which module failed the tests This also fixes the problem that the testsuite threw `exitFailure` even when it succeeded (which I found inexplicable). Once this is published, the testsuite could be enabled in Stackage again. --- .gitignore | 1 + test/shellcheck.hs | 37 ++++++++++++++++++++----------------- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/.gitignore b/.gitignore index 6d5f1ae..cf373a8 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ cabal.config /parts/ /prime/ *.snap +/dist-newstyle/ diff --git a/test/shellcheck.hs b/test/shellcheck.hs index 1a272af..d5e056d 100644 --- a/test/shellcheck.hs +++ b/test/shellcheck.hs @@ -18,21 +18,24 @@ import qualified ShellCheck.Parser main = do putStrLn "Running ShellCheck tests..." - results <- sequence [ - ShellCheck.Analytics.runTests - ,ShellCheck.AnalyzerLib.runTests - ,ShellCheck.ASTLib.runTests - ,ShellCheck.CFG.runTests - ,ShellCheck.CFGAnalysis.runTests - ,ShellCheck.Checker.runTests - ,ShellCheck.Checks.Commands.runTests - ,ShellCheck.Checks.ControlFlow.runTests - ,ShellCheck.Checks.Custom.runTests - ,ShellCheck.Checks.ShellSupport.runTests - ,ShellCheck.Fixer.runTests - ,ShellCheck.Formatter.Diff.runTests - ,ShellCheck.Parser.runTests + failures <- filter (not . snd) <$> mapM sequenceA tests + if null failures then exitSuccess else do + putStrLn "Tests failed for the following module(s):" + mapM (putStrLn . ("- ShellCheck." ++) . fst) failures + exitFailure + where + tests = + [ ("Analytics" , ShellCheck.Analytics.runTests) + , ("AnalyzerLib" , ShellCheck.AnalyzerLib.runTests) + , ("ASTLib" , ShellCheck.ASTLib.runTests) + , ("CFG" , ShellCheck.CFG.runTests) + , ("CFGAnalysis" , ShellCheck.CFGAnalysis.runTests) + , ("Checker" , ShellCheck.Checker.runTests) + , ("Checks.Commands" , ShellCheck.Checks.Commands.runTests) + , ("Checks.ControlFlow" , ShellCheck.Checks.ControlFlow.runTests) + , ("Checks.Custom" , ShellCheck.Checks.Custom.runTests) + , ("Checks.ShellSupport", ShellCheck.Checks.ShellSupport.runTests) + , ("Fixer" , ShellCheck.Fixer.runTests) + , ("Formatter.Diff" , ShellCheck.Formatter.Diff.runTests) + , ("Parser" , ShellCheck.Parser.runTests) ] - if and results - then exitSuccess - else exitFailure From 74282b0a9319bdcc0e3a4a17e2db1969e1c73be2 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 10 Dec 2023 17:05:29 -0800 Subject: [PATCH 336/459] Recognize 'busybox' in --shell and directives. Add to doc texts. --- shellcheck.1.md | 3 ++- shellcheck.hs | 2 +- src/ShellCheck/Checks/ShellSupport.hs | 2 ++ src/ShellCheck/Data.hs | 1 + 4 files changed, 6 insertions(+), 2 deletions(-) diff --git a/shellcheck.1.md b/shellcheck.1.md index 9675e79..89f6d50 100644 --- a/shellcheck.1.md +++ b/shellcheck.1.md @@ -85,7 +85,8 @@ not warn at all, as `ksh` supports decimals in arithmetic contexts. **-s**\ *shell*,\ **--shell=***shell* -: Specify Bourne shell dialect. Valid values are *sh*, *bash*, *dash* and *ksh*. +: Specify Bourne shell dialect. Valid values are *sh*, *bash*, *dash*, *ksh*, + and *busybox*. The default is to deduce the shell from the file's `shell` directive, shebang, or `.bash/.bats/.dash/.ksh` extension, in that order. *sh* refers to POSIX `sh` (not the system's), and will warn of portability issues. diff --git a/shellcheck.hs b/shellcheck.hs index 6be9bb1..6f12238 100644 --- a/shellcheck.hs +++ b/shellcheck.hs @@ -115,7 +115,7 @@ options = [ "Specify path when looking for sourced files (\"SCRIPTDIR\" for script's dir)", Option "s" ["shell"] (ReqArg (Flag "shell") "SHELLNAME") - "Specify dialect (sh, bash, dash, ksh)", + "Specify dialect (sh, bash, dash, ksh, busybox)", Option "S" ["severity"] (ReqArg (Flag "severity") "SEVERITY") "Minimum severity of errors to consider (error, warning, info, style)", diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index e652cb5..d070497 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -211,6 +211,8 @@ prop_checkBashisms116 = verify checkBashisms "#!/bin/busybox sh\nx='test'\n${x[1 prop_checkBashisms117 = verify checkBashisms "#!/bin/busybox sh\nx='test'\n${!x[@]}" -- SC3055 prop_checkBashisms118 = verify checkBashisms "#!/bin/busybox sh\nxyz=1\n${!x*}" -- SC3056 prop_checkBashisms119 = verify checkBashisms "#!/bin/busybox sh\nx='test'\n${x^^[t]}" -- SC3059 +prop_checkBashisms120 = verify checkBashisms "#!/bin/sh\n[ x == y ]" +prop_checkBashisms121 = verifyNot checkBashisms "#!/bin/sh\n# shellcheck shell=busybox\n[ x == y ]" checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do params <- ask kludge params t diff --git a/src/ShellCheck/Data.hs b/src/ShellCheck/Data.hs index 6a87123..917142e 100644 --- a/src/ShellCheck/Data.hs +++ b/src/ShellCheck/Data.hs @@ -156,6 +156,7 @@ shellForExecutable name = "sh" -> return Sh "bash" -> return Bash "bats" -> return Bash + "busybox" -> return BusyboxSh -- Used for directives and --shell=busybox "busybox sh" -> return BusyboxSh "busybox ash" -> return BusyboxSh "dash" -> return Dash From f2729f73cbffde5ef332ce943bc07021302781ab Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 10 Dec 2023 17:57:33 -0800 Subject: [PATCH 337/459] Abuse STRIP to avoid crashes on unsupported AST nodes --- src/ShellCheck/CFG.hs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/CFG.hs b/src/ShellCheck/CFG.hs index 2fe11e7..0cd6326 100644 --- a/src/ShellCheck/CFG.hs +++ b/src/ShellCheck/CFG.hs @@ -887,7 +887,9 @@ build t = do T_Less _ -> none T_ParamSubSpecialChar _ _ -> none - x -> error ("Unimplemented: " ++ show x) + x -> do + error ("Unimplemented: " ++ show x) -- STRIP + none -- Still in `where` clause forInHelper id name words body = do From a9e7bf1950ed50bdfbc818710085c6414f8bf20e Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 10 Dec 2023 19:13:34 -0800 Subject: [PATCH 338/459] Reparse indices after attaching here docs (fixes #2846) --- src/ShellCheck/Checker.hs | 3 +++ src/ShellCheck/Parser.hs | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/ShellCheck/Checker.hs b/src/ShellCheck/Checker.hs index c79f90f..6c9166f 100644 --- a/src/ShellCheck/Checker.hs +++ b/src/ShellCheck/Checker.hs @@ -516,6 +516,9 @@ prop_hereDocsAreParsedWithoutTrailingLinefeed = 1044 `elem` result where result = check "cat << eof" +prop_hereDocsWillHaveParsedIndices = null result + where + result = check "#!/bin/bash\nmy_array=(a b)\ncat <> ./test\n $(( 1 + my_array[1] ))\nEOF" return [] runTests = $quickCheckAll diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index abd4d94..701010f 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -3339,7 +3339,8 @@ readScriptFile sourced = do verifyEof let script = T_Annotation annotationId annotations $ T_Script id shebang commands - reparseIndices script + userstate <- getState + reparseIndices $ reattachHereDocs script (hereDocMap userstate) else do many anyChar id <- endSpan start @@ -3487,8 +3488,7 @@ parseShell env name contents = do return newParseResult { prComments = map toPositionedComment $ nub $ parseNotes userstate ++ parseProblems state, prTokenPositions = Map.map startEndPosToPos (positionMap userstate), - prRoot = Just $ - reattachHereDocs script (hereDocMap userstate) + prRoot = Just script } Left err -> do let context = contextStack state From 4c1d9171b26074e52736f3a58635d69a18326d68 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Mon, 11 Dec 2023 15:08:39 -0500 Subject: [PATCH 339/459] Remove partial head function from src/ShellCheck/Formatter/TTY.hs --- src/ShellCheck/Formatter/TTY.hs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/ShellCheck/Formatter/TTY.hs b/src/ShellCheck/Formatter/TTY.hs index e28696c..e503639 100644 --- a/src/ShellCheck/Formatter/TTY.hs +++ b/src/ShellCheck/Formatter/TTY.hs @@ -31,9 +31,9 @@ import Data.Ord import Data.IORef import Data.List import Data.Maybe -import GHC.Exts import System.IO import System.Info +import qualified Data.List.NonEmpty as NE wikiLink = "https://www.shellcheck.net/wiki/" @@ -117,19 +117,19 @@ outputResult options ref result sys = do color <- getColorFunc $ foColorOption options let comments = crComments result appendComments ref comments (fromIntegral $ foWikiLinkCount options) - let fileGroups = groupWith sourceFile comments + let fileGroups = NE.groupWith sourceFile comments mapM_ (outputForFile color sys) fileGroups outputForFile color sys comments = do - let fileName = sourceFile (head comments) + let fileName = sourceFile (NE.head comments) result <- siReadFile sys (Just True) fileName let contents = either (const "") id result let fileLinesList = lines contents let lineCount = length fileLinesList let fileLines = listArray (1, lineCount) fileLinesList - let groups = groupWith lineNo comments + let groups = NE.groupWith lineNo comments forM_ groups $ \commentsForLine -> do - let lineNum = fromIntegral $ lineNo (head commentsForLine) + let lineNum = fromIntegral $ lineNo (NE.head commentsForLine) let line = if lineNum < 1 || lineNum > lineCount then "" else fileLines ! fromIntegral lineNum @@ -139,7 +139,7 @@ outputForFile color sys comments = do putStrLn (color "source" line) forM_ commentsForLine $ \c -> putStrLn $ color (severityText c) $ cuteIndent c putStrLn "" - showFixedString color commentsForLine (fromIntegral lineNum) fileLines + showFixedString color (toList commentsForLine) (fromIntegral lineNum) fileLines -- Pick out only the lines necessary to show a fix in action sliceFile :: Fix -> Array Int String -> (Fix, Array Int String) From e5208ccb50e3d10957c13f0b77d19936fa4842e1 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Mon, 11 Dec 2023 15:43:35 -0500 Subject: [PATCH 340/459] Remove partial head function from src/ShellCheck/Formatter/JSON1.hs --- src/ShellCheck/Formatter/JSON1.hs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/Formatter/JSON1.hs b/src/ShellCheck/Formatter/JSON1.hs index 2169bf6..b4dbe35 100644 --- a/src/ShellCheck/Formatter/JSON1.hs +++ b/src/ShellCheck/Formatter/JSON1.hs @@ -27,9 +27,9 @@ import Control.DeepSeq import Data.Aeson import Data.IORef import Data.Monoid -import GHC.Exts import System.IO import qualified Data.ByteString.Lazy.Char8 as BL +import qualified Data.List.NonEmpty as NE format :: IO Formatter format = do @@ -114,10 +114,10 @@ outputError file msg = hPutStrLn stderr $ file ++ ": " ++ msg collectResult ref cr sys = mapM_ f groups where comments = crComments cr - groups = groupWith sourceFile comments - f :: [PositionedComment] -> IO () + groups = NE.groupWith sourceFile comments + f :: NE.NonEmpty PositionedComment -> IO () f group = do - let filename = sourceFile (head group) + let filename = sourceFile (NE.head group) result <- siReadFile sys (Just True) filename let contents = either (const "") id result let comments' = makeNonVirtual comments contents From 5a961371a75baea7d04ce96bf8b85e548f566e1a Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Mon, 11 Dec 2023 15:55:29 -0500 Subject: [PATCH 341/459] Remove partial head function from src/ShellCheck/Formatter/GCC.hs --- src/ShellCheck/Formatter/GCC.hs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/ShellCheck/Formatter/GCC.hs b/src/ShellCheck/Formatter/GCC.hs index 5106e4c..b921753 100644 --- a/src/ShellCheck/Formatter/GCC.hs +++ b/src/ShellCheck/Formatter/GCC.hs @@ -23,8 +23,8 @@ import ShellCheck.Interface import ShellCheck.Formatter.Format import Data.List -import GHC.Exts import System.IO +import qualified Data.List.NonEmpty as NE format :: IO Formatter format = return Formatter { @@ -39,13 +39,13 @@ outputError file error = hPutStrLn stderr $ file ++ ": " ++ error outputAll cr sys = mapM_ f groups where comments = crComments cr - groups = groupWith sourceFile comments - f :: [PositionedComment] -> IO () + groups = NE.groupWith sourceFile comments + f :: NE.NonEmpty PositionedComment -> IO () f group = do - let filename = sourceFile (head group) + let filename = sourceFile (NE.head group) result <- siReadFile sys (Just True) filename let contents = either (const "") id result - outputResult filename contents group + outputResult filename contents (NE.toList group) outputResult filename contents warnings = do let comments = makeNonVirtual warnings contents From e5028481e24e6e95ee11c6fae2323fca80449700 Mon Sep 17 00:00:00 2001 From: slycordinator <68940237+slycordinator@users.noreply.github.com> Date: Thu, 14 Dec 2023 15:24:49 +0900 Subject: [PATCH 342/459] Add installation directions for winge ShellCheck is now available on winget, so we can add it to the installation methods. --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 9d98b9d..ca9b847 100644 --- a/README.md +++ b/README.md @@ -194,6 +194,12 @@ On Windows (via [chocolatey](https://chocolatey.org/packages/shellcheck)): C:\> choco install shellcheck ``` +Or Windows (via [winget](https://github.com/microsoft/winget-pkgs)): + +```cmd +C:\> winget install --id koalaman.shellcheck +``` + Or Windows (via [scoop](http://scoop.sh)): ```cmd From 09d04c4c9b7dc0c8b466a6976d901ef1fc5c52e5 Mon Sep 17 00:00:00 2001 From: Jens Petersen Date: Fri, 15 Dec 2023 22:40:48 +0800 Subject: [PATCH 343/459] .cabal: allow Diff-0.5 --- ShellCheck.cabal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ShellCheck.cabal b/ShellCheck.cabal index 76516db..a12f75e 100644 --- a/ShellCheck.cabal +++ b/ShellCheck.cabal @@ -53,7 +53,7 @@ library bytestring >= 0.10.6 && < 0.13, containers >= 0.5.6 && < 0.8, deepseq >= 1.4.1 && < 1.6, - Diff >= 0.4.0 && < 0.5, + Diff >= 0.4.0 && < 0.6, fgl (>= 5.7.0 && < 5.8.1.0) || (>= 5.8.1.1 && < 5.9), filepath >= 1.4.0 && < 1.5, mtl >= 2.2.2 && < 2.4, From a37803d2b873329f062788f5bcfd20fca8f45edc Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Mon, 18 Dec 2023 23:57:47 -0500 Subject: [PATCH 344/459] Remove partial head function from src/ShellCheck/Formatter/CheckStyle.hs --- src/ShellCheck/Formatter/CheckStyle.hs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/Formatter/CheckStyle.hs b/src/ShellCheck/Formatter/CheckStyle.hs index 6ad6c9c..3f898c3 100644 --- a/src/ShellCheck/Formatter/CheckStyle.hs +++ b/src/ShellCheck/Formatter/CheckStyle.hs @@ -24,8 +24,8 @@ import ShellCheck.Formatter.Format import Data.Char import Data.List -import GHC.Exts import System.IO +import qualified Data.List.NonEmpty as NE format :: IO Formatter format = return Formatter { @@ -45,12 +45,12 @@ outputResults cr sys = else mapM_ outputGroup fileGroups where comments = crComments cr - fileGroups = groupWith sourceFile comments + fileGroups = NE.groupWith sourceFile comments outputGroup group = do - let filename = sourceFile (head group) + let filename = sourceFile (NE.head group) result <- siReadFile sys (Just True) filename let contents = either (const "") id result - outputFile filename contents group + outputFile filename contents (NE.toList group) outputFile filename contents warnings = do let comments = makeNonVirtual warnings contents From f242922a2e76ad761f23ffba9040293811e7862a Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Tue, 19 Dec 2023 00:00:32 -0500 Subject: [PATCH 345/459] Use onlyLiteralString in more places --- src/ShellCheck/Analytics.hs | 6 +++--- src/ShellCheck/Checks/Commands.hs | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 108682a..b7844dd 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1443,14 +1443,14 @@ prop_checkConstantNullary5 = verify checkConstantNullary "[[ true ]]" prop_checkConstantNullary6 = verify checkConstantNullary "[ 1 ]" prop_checkConstantNullary7 = verify checkConstantNullary "[ false ]" checkConstantNullary _ (TC_Nullary _ _ t) | isConstant t = - case fromMaybe "" $ getLiteralString t of + case onlyLiteralString t of "false" -> err (getId t) 2158 "[ false ] is true. Remove the brackets." "0" -> err (getId t) 2159 "[ 0 ] is true. Use 'false' instead." "true" -> style (getId t) 2160 "Instead of '[ true ]', just use 'true'." "1" -> style (getId t) 2161 "Instead of '[ 1 ]', use 'true'." _ -> err (getId t) 2078 "This expression is constant. Did you forget a $ somewhere?" where - string = fromMaybe "" $ getLiteralString t + string = onlyLiteralString t checkConstantNullary _ _ = return () @@ -2276,7 +2276,7 @@ checkFunctionsUsedExternally params t = (Just str, t) -> do let name = basename str let args = skipOver t argv - let argStrings = map (\x -> (fromMaybe "" $ getLiteralString x, x)) args + let argStrings = map (\x -> (onlyLiteralString x, x)) args let candidates = getPotentialCommands name argStrings mapM_ (checkArg name (getId t)) candidates _ -> return () diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 8be60a7..86fda24 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -186,7 +186,7 @@ checkCommand map t@(T_SimpleCommand id cmdPrefix (cmd:rest)) = sequence_ $ do M.findWithDefault nullCheck (Basename $ basename name) map t else if name == "builtin" && not (null rest) then let t' = T_SimpleCommand id cmdPrefix rest - selectedBuiltin = fromMaybe "" $ getLiteralString . head $ rest + selectedBuiltin = onlyLiteralString $ head rest in M.findWithDefault nullCheck (Exactly selectedBuiltin) map t' else do M.findWithDefault nullCheck (Exactly name) map t @@ -299,7 +299,7 @@ checkExpr = CommandCheck (Basename "expr") f where "'expr' expects 3+ arguments but sees 1. Make sure each operator/operand is a separate argument, and escape <>&|." [first, second] | - (fromMaybe "" $ getLiteralString first) /= "length" + onlyLiteralString first /= "length" && not (willSplit first || willSplit second) -> do checkOp first warn (getId t) 2307 From c97abdb939a52beefa576d984b104eb89e7667d9 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Tue, 19 Dec 2023 00:41:12 -0500 Subject: [PATCH 346/459] Make HereDocPending only hold the relevant pieces of a T_HereDoc instead of an arbitrary Token --- src/ShellCheck/Parser.hs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 701010f..0e2fc6d 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -160,7 +160,7 @@ data Context = deriving (Show) data HereDocContext = - HereDocPending Token [Context] -- on linefeed, read this T_HereDoc + HereDocPending Id Dashed Quoted String [Context] -- on linefeed, read this T_HereDoc deriving (Show) data UserState = UserState { @@ -238,12 +238,12 @@ addToHereDocMap id list = do hereDocMap = Map.insert id list map } -addPendingHereDoc t = do +addPendingHereDoc id d q str = do state <- getState context <- getCurrentContexts let docs = pendingHereDocs state putState $ state { - pendingHereDocs = HereDocPending t context : docs + pendingHereDocs = HereDocPending id d q str context : docs } popPendingHereDocs = do @@ -1835,7 +1835,7 @@ readHereDoc = called "here document" $ do -- add empty tokens for now, read the rest in readPendingHereDocs let doc = T_HereDoc hid dashed quoted endToken [] - addPendingHereDoc doc + addPendingHereDoc hid dashed quoted endToken return doc where unquote :: String -> (Quoted, String) @@ -1856,7 +1856,7 @@ readPendingHereDocs = do docs <- popPendingHereDocs mapM_ readDoc docs where - readDoc (HereDocPending (T_HereDoc id dashed quoted endToken _) ctx) = + readDoc (HereDocPending id dashed quoted endToken ctx) = swapContext ctx $ do docStartPos <- getPosition From c1452e0d174fe6c4c0a3775d6ee430de1bd8ccda Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Tue, 19 Dec 2023 00:53:08 -0500 Subject: [PATCH 347/459] Remove unnecessary partiality from kludgeAwayQuotes --- src/ShellCheck/Parser.hs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 0e2fc6d..115de64 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -46,6 +46,7 @@ import Text.Parsec.Error import Text.Parsec.Pos import qualified Control.Monad.Reader as Mr import qualified Control.Monad.State as Ms +import qualified Data.List.NonEmpty as NE import qualified Data.Map.Strict as Map import Test.QuickCheck.All (quickCheckAll) @@ -2904,8 +2905,8 @@ readLetSuffix = many1 (readIoRedirect <|> try readLetExpression <|> readCmdWord) kludgeAwayQuotes :: String -> SourcePos -> (String, SourcePos) kludgeAwayQuotes s p = case s of - first:rest@(_:_) -> - let (last:backwards) = reverse rest + first:second:rest -> + let (last NE.:| backwards) = NE.reverse (second NE.:| rest) middle = reverse backwards in if first `elem` "'\"" && first == last From 208e38358e8c07688a29867235971136a8ed0092 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Tue, 19 Dec 2023 01:00:20 -0500 Subject: [PATCH 348/459] Use a list comprehension to remove partiality from notesForContext --- src/ShellCheck/Parser.hs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 115de64..04bdbc4 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -3507,13 +3507,11 @@ parseShell env name contents = do -- A final pass for ignoring parse errors after failed parsing isIgnored stack note = any (contextItemDisablesCode False (codeForParseNote note)) stack -notesForContext list = zipWith ($) [first, second] $ filter isName list +notesForContext list = zipWith ($) [first, second] [(pos, str) | ContextName pos str <- list] where - isName (ContextName _ _) = True - isName _ = False - first (ContextName pos str) = ParseNote pos pos ErrorC 1073 $ + first (pos, str) = ParseNote pos pos ErrorC 1073 $ "Couldn't parse this " ++ str ++ ". Fix to allow more checks." - second (ContextName pos str) = ParseNote pos pos InfoC 1009 $ + second (pos, str) = ParseNote pos pos InfoC 1009 $ "The mentioned syntax error was in this " ++ str ++ "." -- Go over all T_UnparsedIndex and reparse them as either arithmetic or text From 0c46b8b2d5dffcb01b5a4689f16b687bc5a5f84e Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Tue, 19 Dec 2023 01:49:04 -0500 Subject: [PATCH 349/459] Use NonEmpty to remove partiality from handleCommand --- src/ShellCheck/CFG.hs | 41 +++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/src/ShellCheck/CFG.hs b/src/ShellCheck/CFG.hs index 0cd6326..5476ad5 100644 --- a/src/ShellCheck/CFG.hs +++ b/src/ShellCheck/CFG.hs @@ -51,6 +51,7 @@ import Control.Monad.Identity import Data.Array.Unboxed import Data.Array.ST import Data.List hiding (map) +import qualified Data.List.NonEmpty as NE import Data.Maybe import qualified Data.Map as M import qualified Data.Set as S @@ -857,8 +858,8 @@ build t = do status <- newNodeRange (CFSetExitCode id) linkRange assignments status - T_SimpleCommand id vars list@(cmd:_) -> - handleCommand t vars list $ getUnquotedLiteral cmd + T_SimpleCommand id vars (cmd:args) -> + handleCommand t vars (cmd NE.:| args) $ getUnquotedLiteral cmd T_SingleQuoted _ _ -> none @@ -925,8 +926,8 @@ handleCommand cmd vars args literalCmd = do -- TODO: Handle assignments in declaring commands case literalCmd of - Just "exit" -> regularExpansion vars args $ handleExit - Just "return" -> regularExpansion vars args $ handleReturn + Just "exit" -> regularExpansion vars (NE.toList args) $ handleExit + Just "return" -> regularExpansion vars (NE.toList args) $ handleReturn Just "unset" -> regularExpansionWithStatus vars args $ handleUnset args Just "declare" -> handleDeclare args @@ -949,14 +950,14 @@ handleCommand cmd vars args literalCmd = do -- This will mostly behave like 'command' but ok Just "builtin" -> case args of - [_] -> regular - (_:newargs@(newcmd:_)) -> - handleCommand newcmd vars newargs $ getLiteralString newcmd + _ NE.:| [] -> regular + (_ NE.:| newcmd:newargs) -> + handleCommand newcmd vars (newcmd NE.:| newargs) $ getLiteralString newcmd Just "command" -> case args of - [_] -> regular - (_:newargs@(newcmd:_)) -> - handleOthers (getId newcmd) vars newargs $ getLiteralString newcmd + _ NE.:| [] -> regular + (_ NE.:| newcmd:newargs) -> + handleOthers (getId newcmd) vars (newcmd NE.:| newargs) $ getLiteralString newcmd _ -> regular where @@ -984,7 +985,7 @@ handleCommand cmd vars args literalCmd = do unreachable <- newNode CFUnreachable return $ Range ret unreachable - handleUnset (cmd:args) = do + handleUnset (cmd NE.:| args) = do case () of _ | "n" `elem` flagNames -> unsetWith CFUndefineNameref _ | "v" `elem` flagNames -> unsetWith CFUndefineVariable @@ -1003,7 +1004,7 @@ handleCommand cmd vars args literalCmd = do variableAssignRegex = mkRegex "^([_a-zA-Z][_a-zA-Z0-9]*)=" - handleDeclare (cmd:args) = do + handleDeclare (cmd NE.:| args) = do isFunc <- asks cfIsFunction -- This is a bit of a kludge: we don't have great support for things like -- 'declare -i x=$x' so do one round with declare x=$x, followed by declare -i x @@ -1092,7 +1093,7 @@ handleCommand cmd vars args literalCmd = do in concatMap (drop 1) plusses - handlePrintf (cmd:args) = + handlePrintf (cmd NE.:| args) = newNodeRange $ CFApplyEffects $ maybeToList findVar where findVar = do @@ -1101,7 +1102,7 @@ handleCommand cmd vars args literalCmd = do name <- getLiteralString arg return $ IdTagged (getId arg) $ CFWriteVariable name CFValueString - handleWait (cmd:args) = + handleWait (cmd NE.:| args) = newNodeRange $ CFApplyEffects $ maybeToList findVar where findVar = do @@ -1110,7 +1111,7 @@ handleCommand cmd vars args literalCmd = do name <- getLiteralString arg return $ IdTagged (getId arg) $ CFWriteVariable name CFValueInteger - handleMapfile (cmd:args) = + handleMapfile (cmd NE.:| args) = newNodeRange $ CFApplyEffects [findVar] where findVar = @@ -1130,7 +1131,7 @@ handleCommand cmd vars args literalCmd = do guard $ isVariableName name return (getId c, name) - handleRead (cmd:args) = newNodeRange $ CFApplyEffects main + handleRead (cmd NE.:| args) = newNodeRange $ CFApplyEffects main where main = fromMaybe fallback $ do flags <- getGnuOpts flagsForRead args @@ -1160,7 +1161,7 @@ handleCommand cmd vars args literalCmd = do in map (\(id, name) -> IdTagged id $ CFWriteVariable name value) namesOrDefault - handleDEFINE (cmd:args) = + handleDEFINE (cmd NE.:| args) = newNodeRange $ CFApplyEffects $ maybeToList findVar where findVar = do @@ -1170,7 +1171,7 @@ handleCommand cmd vars args literalCmd = do return $ IdTagged (getId name) $ CFWriteVariable str CFValueString handleOthers id vars args cmd = - regularExpansion vars args $ do + regularExpansion vars (NE.toList args) $ do exe <- newNodeRange $ CFExecuteCommand cmd status <- newNodeRange $ CFSetExitCode id linkRange exe status @@ -1189,8 +1190,8 @@ handleCommand cmd vars args literalCmd = do linkRanges $ [args] ++ assignments ++ [exe] ++ dropAssignments - regularExpansionWithStatus vars args@(cmd:_) p = do - initial <- regularExpansion vars args p + regularExpansionWithStatus vars args@(cmd NE.:| _) p = do + initial <- regularExpansion vars (NE.toList args) p status <- newNodeRange $ CFSetExitCode (getId cmd) linkRange initial status From eed0174e90a374ee497aef7d7649e022856e80e9 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Tue, 19 Dec 2023 02:06:45 -0500 Subject: [PATCH 350/459] Make "Unresolved scope in dependency" impossible --- src/ShellCheck/CFG.hs | 24 ++++++++++++------------ src/ShellCheck/CFGAnalysis.hs | 17 ++++++++--------- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/src/ShellCheck/CFG.hs b/src/ShellCheck/CFG.hs index 5476ad5..ed6a8f8 100644 --- a/src/ShellCheck/CFG.hs +++ b/src/ShellCheck/CFG.hs @@ -112,8 +112,8 @@ data CFEdge = -- Actions we track data CFEffect = - CFSetProps Scope String (S.Set CFVariableProp) - | CFUnsetProps Scope String (S.Set CFVariableProp) + CFSetProps (Maybe Scope) String (S.Set CFVariableProp) + | CFUnsetProps (Maybe Scope) String (S.Set CFVariableProp) | CFReadVariable String | CFWriteVariable String CFValue | CFWriteGlobal String CFValue @@ -579,7 +579,7 @@ build t = do T_Array _ list -> sequentially list - T_Assignment {} -> buildAssignment DefaultScope t + T_Assignment {} -> buildAssignment Nothing t T_Backgrounded id body -> do start <- newStructuralNode @@ -1031,9 +1031,9 @@ handleCommand cmd vars args literalCmd = do scope isFunc = case () of - _ | global -> GlobalScope - _ | isFunc -> LocalScope - _ -> DefaultScope + _ | global -> Just GlobalScope + _ | isFunc -> Just LocalScope + _ -> Nothing addedProps = S.fromList $ concat $ [ [ CFVPArray | array ], @@ -1178,7 +1178,7 @@ handleCommand cmd vars args literalCmd = do regularExpansion vars args p = do args <- sequentially args - assignments <- mapM (buildAssignment PrefixScope) vars + assignments <- mapM (buildAssignment (Just PrefixScope)) vars exe <- p dropAssignments <- if null vars @@ -1198,7 +1198,7 @@ handleCommand cmd vars args literalCmd = do none = newStructuralNode -data Scope = DefaultScope | GlobalScope | LocalScope | PrefixScope +data Scope = GlobalScope | LocalScope | PrefixScope deriving (Eq, Ord, Show, Generic, NFData) buildAssignment scope t = do @@ -1212,10 +1212,10 @@ buildAssignment scope t = do let valueType = if null indices then f id value else CFValueArray let scoper = case scope of - PrefixScope -> CFWritePrefix - LocalScope -> CFWriteLocal - GlobalScope -> CFWriteGlobal - DefaultScope -> CFWriteVariable + Just PrefixScope -> CFWritePrefix + Just LocalScope -> CFWriteLocal + Just GlobalScope -> CFWriteGlobal + Nothing -> CFWriteVariable write <- newNodeRange $ applySingle $ IdTagged id $ scoper var valueType linkRanges [expand, index, read, write] where diff --git a/src/ShellCheck/CFGAnalysis.hs b/src/ShellCheck/CFGAnalysis.hs index 3b4f957..16afa68 100644 --- a/src/ShellCheck/CFGAnalysis.hs +++ b/src/ShellCheck/CFGAnalysis.hs @@ -299,7 +299,6 @@ depsToState set = foldl insert newInternalState $ S.toList set PrefixScope -> (sPrefixValues, insertPrefix) LocalScope -> (sLocalValues, insertLocal) GlobalScope -> (sGlobalValues, insertGlobal) - DefaultScope -> error $ pleaseReport "Unresolved scope in dependency" alreadyExists = isJust $ vmLookup name $ mapToCheck state in @@ -1120,34 +1119,34 @@ transferEffect ctx effect = CFSetProps scope name props -> case scope of - DefaultScope -> do + Nothing -> do state <- readVariable ctx name writeVariable ctx name $ addProperties props state - GlobalScope -> do + Just GlobalScope -> do state <- readGlobal ctx name writeGlobal ctx name $ addProperties props state - LocalScope -> do + Just LocalScope -> do out <- readSTRef (cOutput ctx) state <- readLocal ctx name writeLocal ctx name $ addProperties props state - PrefixScope -> do + Just PrefixScope -> do -- Prefix values become local state <- readLocal ctx name writeLocal ctx name $ addProperties props state CFUnsetProps scope name props -> case scope of - DefaultScope -> do + Nothing -> do state <- readVariable ctx name writeVariable ctx name $ removeProperties props state - GlobalScope -> do + Just GlobalScope -> do state <- readGlobal ctx name writeGlobal ctx name $ removeProperties props state - LocalScope -> do + Just LocalScope -> do out <- readSTRef (cOutput ctx) state <- readLocal ctx name writeLocal ctx name $ removeProperties props state - PrefixScope -> do + Just PrefixScope -> do -- Prefix values become local state <- readLocal ctx name writeLocal ctx name $ removeProperties props state From a47a42cb45e68050b84122a4bf237502b642350b Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Tue, 19 Dec 2023 02:17:59 -0500 Subject: [PATCH 351/459] Remove unnecessary partiality from isAssignmentParamToCommand --- src/ShellCheck/AnalyzerLib.hs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 4990822..1d53a98 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -341,9 +341,9 @@ isQuoteFreeNode strict shell tree t = -- Is this node self-quoting in itself? isQuoteFreeElement t = case t of - T_Assignment {} -> assignmentIsQuoting t - T_FdRedirect {} -> True - _ -> False + T_Assignment id _ _ _ _ -> assignmentIsQuoting id + T_FdRedirect {} -> True + _ -> False -- Are any subnodes inherently self-quoting? isQuoteFreeContext t = @@ -353,7 +353,7 @@ isQuoteFreeNode strict shell tree t = TC_Binary _ DoubleBracket _ _ _ -> return True TA_Sequence {} -> return True T_Arithmetic {} -> return True - T_Assignment {} -> return $ assignmentIsQuoting t + T_Assignment id _ _ _ _ -> return $ assignmentIsQuoting id T_Redirecting {} -> return False T_DoubleQuoted _ _ -> return True T_DollarDoubleQuoted _ _ -> return True @@ -368,11 +368,11 @@ isQuoteFreeNode strict shell tree t = -- Check whether this assignment is self-quoting due to being a recognized -- assignment passed to a Declaration Utility. This will soon be required -- by POSIX: https://austingroupbugs.net/view.php?id=351 - assignmentIsQuoting t = shellParsesParamsAsAssignments || not (isAssignmentParamToCommand t) + assignmentIsQuoting id = shellParsesParamsAsAssignments || not (isAssignmentParamToCommand id) shellParsesParamsAsAssignments = shell /= Sh -- Is this assignment a parameter to a command like export/typeset/etc? - isAssignmentParamToCommand (T_Assignment id _ _ _ _) = + isAssignmentParamToCommand id = case Map.lookup id tree of Just (T_SimpleCommand _ _ (_:args)) -> id `elem` (map getId args) _ -> False From bfe4342697292fe25a9214e30a0770fe0237ec42 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Tue, 19 Dec 2023 02:30:48 -0500 Subject: [PATCH 352/459] Remove unnecessary partiality from check --- src/ShellCheck/Checks/Commands.hs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 86fda24..314c1e9 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -1006,7 +1006,7 @@ checkWhileGetoptsCase = CommandCheck (Exactly "getopts") f options <- getLiteralString arg1 getoptsVar <- getLiteralString name (T_WhileExpression _ _ body) <- findFirst whileLoop path - caseCmd@(T_CaseExpression _ var _) <- mapMaybe findCase body !!! 0 + T_CaseExpression id var list <- mapMaybe findCase body !!! 0 -- Make sure getopts name and case variable matches [T_DollarBraced _ _ bracedWord] <- return $ getWordParts var @@ -1016,11 +1016,11 @@ checkWhileGetoptsCase = CommandCheck (Exactly "getopts") f -- Make sure the variable isn't modified guard . not $ modifiesVariable params (T_BraceGroup (Id 0) body) getoptsVar - return $ check (getId arg1) (map (:[]) $ filter (/= ':') options) caseCmd + return $ check (getId arg1) (map (:[]) $ filter (/= ':') options) id list f _ = return () - check :: Id -> [String] -> Token -> Analysis - check optId opts (T_CaseExpression id _ list) = do + check :: Id -> [String] -> Id -> [(CaseType, [Token], [Token])] -> Analysis + check optId opts id list = do unless (Nothing `M.member` handledMap) $ do mapM_ (warnUnhandled optId id) $ catMaybes $ M.keys notHandled From f983d9ae93e5eda28297a93a43640aafafa5f46c Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Thu, 21 Dec 2023 13:35:22 -0500 Subject: [PATCH 353/459] Simplify functionMap and remove unnecessary partiality --- src/ShellCheck/Analytics.hs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index b7844dd..19ff51b 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2825,13 +2825,11 @@ checkUnpassedInFunctions params root = execWriter $ mapM_ warnForGroup referenceGroups where functionMap :: Map.Map String Token - functionMap = Map.fromList $ - map (\t@(T_Function _ _ _ name _) -> (name,t)) functions - functions = execWriter $ doAnalysis (tell . maybeToList . findFunction) root + functionMap = Map.fromList $ execWriter $ doAnalysis (tell . maybeToList . findFunction) root findFunction t@(T_Function id _ _ name body) | any (isPositionalReference t) flow && not (any isPositionalAssignment flow) - = return t + = return (name,t) where flow = getVariableFlow params body findFunction _ = Nothing From dab77b2c8d978534603000e5406e604a79e1b195 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Thu, 21 Dec 2023 13:48:47 -0500 Subject: [PATCH 354/459] Implement parseEnum in terms of lookup --- shellcheck.hs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/shellcheck.hs b/shellcheck.hs index 6f12238..00b699b 100644 --- a/shellcheck.hs +++ b/shellcheck.hs @@ -252,9 +252,9 @@ runFormatter sys format options files = do else SomeProblems parseEnum name value list = - case filter ((== value) . fst) list of - [(name, value)] -> return value - [] -> do + case lookup value list of + Just value -> return value + Nothing -> do printErr $ "Unknown value for --" ++ name ++ ". " ++ "Valid options are: " ++ (intercalate ", " $ map fst list) throwError SupportFailure From 3bd7df955bdd9f066d5d19a90712b81305c61c87 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Fri, 29 Dec 2023 14:18:42 -0500 Subject: [PATCH 355/459] Use a pattern match instead of null and head in checkCommand --- src/ShellCheck/Checks/Commands.hs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 314c1e9..429e786 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -20,6 +20,7 @@ {-# LANGUAGE TemplateHaskell #-} {-# LANGUAGE FlexibleContexts #-} {-# LANGUAGE MultiWayIf #-} +{-# LANGUAGE PatternGuards #-} -- This module contains checks that examine specific commands by name. module ShellCheck.Checks.Commands (checker, optionalChecks, ShellCheck.Checks.Commands.runTests) where @@ -181,16 +182,15 @@ checkCommand :: M.Map CommandName (Token -> Analysis) -> Token -> Analysis checkCommand map t@(T_SimpleCommand id cmdPrefix (cmd:rest)) = sequence_ $ do name <- getLiteralString cmd return $ - if '/' `elem` name - then - M.findWithDefault nullCheck (Basename $ basename name) map t - else if name == "builtin" && not (null rest) then - let t' = T_SimpleCommand id cmdPrefix rest - selectedBuiltin = onlyLiteralString $ head rest - in M.findWithDefault nullCheck (Exactly selectedBuiltin) map t' - else do - M.findWithDefault nullCheck (Exactly name) map t - M.findWithDefault nullCheck (Basename name) map t + if | '/' `elem` name -> + M.findWithDefault nullCheck (Basename $ basename name) map t + | name == "builtin", (h:_) <- rest -> + let t' = T_SimpleCommand id cmdPrefix rest + selectedBuiltin = onlyLiteralString h + in M.findWithDefault nullCheck (Exactly selectedBuiltin) map t' + | otherwise -> do + M.findWithDefault nullCheck (Exactly name) map t + M.findWithDefault nullCheck (Basename name) map t where basename = reverse . takeWhile (/= '/') . reverse From dedf932fe8b9dcf4f852d4289032e6d25e7e5d40 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sat, 30 Dec 2023 13:59:15 -0500 Subject: [PATCH 356/459] Use traverse instead of sequence and map --- src/ShellCheck/Analytics.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 19ff51b..df0b3e6 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -4936,7 +4936,7 @@ checkOverwrittenExitCode params t = guard . not $ S.null exitCodeIds let idToToken = idMap params - exitCodeTokens <- sequence $ map (\k -> Map.lookup k idToToken) $ S.toList exitCodeIds + exitCodeTokens <- traverse (\k -> Map.lookup k idToToken) $ S.toList exitCodeIds return $ do when (all isCondition exitCodeTokens && not (usedUnconditionally t exitCodeIds)) $ warn id 2319 "This $? refers to a condition, not a command. Assign to a variable to avoid it being overwritten." From 980e7d3ca8aa19977ca517b846e2d2cff4fb1c5d Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sat, 30 Dec 2023 14:49:26 -0500 Subject: [PATCH 357/459] Use <$> instead of >>= and return --- src/ShellCheck/CFG.hs | 2 +- src/ShellCheck/Parser.hs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/CFG.hs b/src/ShellCheck/CFG.hs index ed6a8f8..5d018c8 100644 --- a/src/ShellCheck/CFG.hs +++ b/src/ShellCheck/CFG.hs @@ -997,7 +997,7 @@ handleCommand cmd vars args literalCmd = do (names, flags) = partition (null . fst) pairs flagNames = map fst flags literalNames :: [(Token, String)] -- Literal names to unset, e.g. [(myfuncToken, "myfunc")] - literalNames = mapMaybe (\(_, t) -> getLiteralString t >>= (return . (,) t)) names + literalNames = mapMaybe (\(_, t) -> (,) t <$> getLiteralString t) names -- Apply a constructor like CFUndefineVariable to each literalName, and tag with its id unsetWith c = newNodeRange $ CFApplyEffects $ map (\(token, name) -> IdTagged (getId token) $ c name) literalNames diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 04bdbc4..37a9b86 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -1195,7 +1195,7 @@ readDollarBracedPart = readSingleQuoted <|> readDoubleQuoted <|> readDollarBracedLiteral = do start <- startSpan - vars <- (readBraceEscaped <|> (anyChar >>= \x -> return [x])) `reluctantlyTill1` bracedQuotable + vars <- (readBraceEscaped <|> ((\x -> [x]) <$> anyChar)) `reluctantlyTill1` bracedQuotable id <- endSpan start return $ T_Literal id $ concat vars @@ -1557,7 +1557,7 @@ readGenericLiteral endChars = do return $ concat strings readGenericLiteral1 endExp = do - strings <- (readGenericEscaped <|> (anyChar >>= \x -> return [x])) `reluctantlyTill1` endExp + strings <- (readGenericEscaped <|> ((\x -> [x]) <$> anyChar)) `reluctantlyTill1` endExp return $ concat strings readGenericEscaped = do @@ -2371,7 +2371,7 @@ readPipeSequence = do return $ T_Pipeline id pipes cmds where sepBy1WithSeparators p s = do - let elems = p >>= \x -> return ([x], []) + let elems = (\x -> ([x], [])) <$> p let seps = do separator <- s return $ \(a,b) (c,d) -> (a++c, b ++ d ++ [separator]) From ee41c780f4a587cd32c58deb8badc27dac2b6b10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Sun, 31 Dec 2023 10:47:40 +0800 Subject: [PATCH 358/459] Replace Atom reference with Pulsar Edit equivalent Since Microsoft acquired GitHub and discontinued Atom in 2022, the community started a fork at https://pulsar-edit.dev/. Linking to an archived repository under the Atom organization does not make sense anymore, so link to active Pulsar fork instead. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9d98b9d..b9aea08 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,7 @@ You can see ShellCheck suggestions directly in a variety of editors. * Sublime, through [SublimeLinter](https://github.com/SublimeLinter/SublimeLinter-shellcheck). -* Atom, through [Linter](https://github.com/AtomLinter/linter-shellcheck). +* Pulsar Edit (former Atom), through [linter-shellcheck-pulsar](https://github.com/pulsar-cooperative/linter-shellcheck-pulsar). * VSCode, through [vscode-shellcheck](https://github.com/timonwong/vscode-shellcheck). From e1ad06383425f46703a26aafc0d1bdc8ddc90a18 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 31 Dec 2023 01:59:53 -0500 Subject: [PATCH 359/459] Implement getPath in terms of unfoldr --- src/ShellCheck/ASTLib.hs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index cf55498..5b3ffd8 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -897,10 +897,9 @@ getUnmodifiedParameterExpansion t = _ -> Nothing --- A list of the element and all its parents up to the root node. -getPath tree t = t : - case Map.lookup (getId t) tree of - Nothing -> [] - Just parent -> getPath tree parent +getPath tree t = t : unfoldr go t + where + go s = (\x -> (x,x)) <$> Map.lookup (getId s) tree isClosingFileOp op = case op of From add49cda171058b921fb57a2b658511159858a1f Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 31 Dec 2023 02:12:58 -0500 Subject: [PATCH 360/459] Make getPath return a NonEmpty --- src/ShellCheck/ASTLib.hs | 5 ++- src/ShellCheck/Analytics.hs | 59 ++++++++++++++++--------------- src/ShellCheck/AnalyzerLib.hs | 11 +++--- src/ShellCheck/Checks/Commands.hs | 3 +- 4 files changed, 40 insertions(+), 38 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 5b3ffd8..aadff05 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -31,6 +31,7 @@ import Data.Functor import Data.Functor.Identity import Data.List import Data.Maybe +import qualified Data.List.NonEmpty as NE import qualified Data.Map as Map import Numeric (showHex) @@ -897,9 +898,7 @@ getUnmodifiedParameterExpansion t = _ -> Nothing --- A list of the element and all its parents up to the root node. -getPath tree t = t : unfoldr go t - where - go s = (\x -> (x,x)) <$> Map.lookup (getId s) tree +getPath tree = NE.unfoldr $ \t -> (t, Map.lookup (getId t) tree) isClosingFileOp op = case op of diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index df0b3e6..e80ed58 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -46,6 +46,7 @@ import Data.Maybe import Data.Ord import Data.Semigroup import Debug.Trace -- STRIP +import qualified Data.List.NonEmpty as NE import qualified Data.Map.Strict as Map import qualified Data.Set as S import Test.QuickCheck.All (forAllProperties) @@ -846,14 +847,14 @@ checkRedirectToSame params s@(T_Pipeline _ _ list) = getRedirs _ = [] special x = "/dev/" `isPrefixOf` concat (oversimplify x) isInput t = - case drop 1 $ getPath (parentMap params) t of + case NE.tail $ getPath (parentMap params) t of T_IoFile _ op _:_ -> case op of T_Less _ -> True _ -> False _ -> False isOutput t = - case drop 1 $ getPath (parentMap params) t of + case NE.tail $ getPath (parentMap params) t of T_IoFile _ op _:_ -> case op of T_Greater _ -> True @@ -887,7 +888,7 @@ checkShorthandIf params x@(T_OrIf _ (T_AndIf id _ _) (T_Pipeline _ _ t)) name <- getCommandBasename t return $ name `elem` ["echo", "exit", "return", "printf"]) isOk _ = False - inCondition = isCondition $ getPath (parentMap params) x + inCondition = isCondition $ NE.toList $ getPath (parentMap params) x checkShorthandIf _ _ = return () @@ -1087,7 +1088,7 @@ checkSingleQuotedVariables params t@(T_SingleQuoted id s) = return $ if name == "find" then getFindCommand cmd else if name == "git" then getGitCommand cmd else if name == "mumps" then getMumpsCommand cmd else name isProbablyOk = - any isOkAssignment (take 3 $ getPath parents t) + any isOkAssignment (NE.take 3 $ getPath parents t) || commandName `elem` [ "trap" ,"sh" @@ -1495,7 +1496,7 @@ checkArithmeticDeref params t@(TA_Expansion _ [T_DollarBraced id _ l]) = where isException [] = True isException s@(h:_) = any (`elem` "/.:#%?*@$-!+=^,") s || isDigit h - getWarning = fromMaybe noWarning . msum . map warningFor $ parents params t + getWarning = fromMaybe noWarning . msum . NE.map warningFor $ parents params t warningFor t = case t of T_Arithmetic {} -> return normalWarning @@ -1823,7 +1824,7 @@ checkInexplicablyUnquoted params (T_NormalWord id tokens) = mapM_ check (tails t T_Literal id s | not (quotesSingleThing a && quotesSingleThing b || s `elem` ["=", ":", "/"] - || isSpecial (getPath (parentMap params) trapped) + || isSpecial (NE.toList $ getPath (parentMap params) trapped) ) -> warnAboutLiteral id _ -> return () @@ -2041,7 +2042,7 @@ doVariableFlowAnalysis readFunc writeFunc empty flow = evalState ( -- from $foo=bar to foo=bar. This is not pretty but ok. quotesMayConflictWithSC2281 params t = case getPath (parentMap params) t of - _ : T_NormalWord parentId (me:T_Literal _ ('=':_):_) : T_SimpleCommand _ _ (cmd:_) : _ -> + _ NE.:| T_NormalWord parentId (me:T_Literal _ ('=':_):_) : T_SimpleCommand _ _ (cmd:_) : _ -> (getId t) == (getId me) && (parentId == getId cmd) _ -> False @@ -2652,7 +2653,7 @@ checkPrefixAssignmentReference params t@(T_DollarBraced id _ value) = check path where name = getBracedReference $ concat $ oversimplify value - path = getPath (parentMap params) t + path = NE.toList $ getPath (parentMap params) t idPath = map getId path check [] = return () @@ -2701,7 +2702,7 @@ checkCharRangeGlob p t@(T_Glob id str) | return $ isCommandMatch cmd (`elem` ["tr", "read"]) -- Check if this is a dereferencing context like [[ -v array[operandhere] ]] - isDereferenced = fromMaybe False . msum . map isDereferencingOp . getPath (parentMap p) + isDereferenced = fromMaybe False . msum . NE.map isDereferencingOp . getPath (parentMap p) isDereferencingOp t = case t of TC_Binary _ DoubleBracket str _ _ -> return $ isDereferencingBinaryOp str @@ -2764,7 +2765,7 @@ checkLoopKeywordScope params t | _ -> return () where name = getCommandName t - path = let p = getPath (parentMap params) t in filter relevant p + path = let p = getPath (parentMap params) t in NE.filter relevant p subshellType t = case leadType params t of NoneScope -> Nothing SubshellScope str -> return str @@ -3188,7 +3189,7 @@ checkUncheckedCdPushdPopd params root = | name `elem` ["cd", "pushd", "popd"] && not (isSafeDir t) && not (name `elem` ["pushd", "popd"] && ("n" `elem` map snd (getAllFlags t))) - && not (isCondition $ getPath (parentMap params) t) = + && not (isCondition $ NE.toList $ getPath (parentMap params) t) = warnWithFix (getId t) 2164 ("Use '" ++ name ++ " ... || exit' or '" ++ name ++ " ... || return' in case " ++ name ++ " fails.") (fixWith [replaceEnd (getId t) params 0 " || exit"]) @@ -3217,7 +3218,7 @@ checkLoopVariableReassignment params token = return $ do warn (getId token) 2165 "This nested loop overrides the index variable of its parent." warn (getId next) 2167 "This parent loop has its index variable overridden." - path = drop 1 $ getPath (parentMap params) token + path = NE.tail $ getPath (parentMap params) token loopVariable :: Token -> Maybe String loopVariable t = case t of @@ -3290,17 +3291,17 @@ checkReturnAgainstZero params token = -- We don't want to warn about composite expressions like -- [[ $? -eq 0 || $? -eq 4 ]] since these can be annoying to rewrite. isOnlyTestInCommand t = - case getPath (parentMap params) t of - _:(T_Condition {}):_ -> True - _:(T_Arithmetic {}):_ -> True - _:(TA_Sequence _ [_]):(T_Arithmetic {}):_ -> True + case NE.tail $ getPath (parentMap params) t of + (T_Condition {}):_ -> True + (T_Arithmetic {}):_ -> True + (TA_Sequence _ [_]):(T_Arithmetic {}):_ -> True -- Some negations and groupings are also fine - _:next@(TC_Unary _ _ "!" _):_ -> isOnlyTestInCommand next - _:next@(TA_Unary _ "!" _):_ -> isOnlyTestInCommand next - _:next@(TC_Group {}):_ -> isOnlyTestInCommand next - _:next@(TA_Sequence _ [_]):_ -> isOnlyTestInCommand next - _:next@(TA_Parentesis _ _):_ -> isOnlyTestInCommand next + next@(TC_Unary _ _ "!" _):_ -> isOnlyTestInCommand next + next@(TA_Unary _ "!" _):_ -> isOnlyTestInCommand next + next@(TC_Group {}):_ -> isOnlyTestInCommand next + next@(TA_Sequence _ [_]):_ -> isOnlyTestInCommand next + next@(TA_Parentesis _ _):_ -> isOnlyTestInCommand next _ -> False -- TODO: Do better $? tracking and filter on whether @@ -3365,7 +3366,7 @@ checkRedirectedNowhere params token = _ -> return () where isInExpansion t = - case drop 1 $ getPath (parentMap params) t of + case NE.tail $ getPath (parentMap params) t of T_DollarExpansion _ [_] : _ -> True T_Backticked _ [_] : _ -> True t@T_Annotation {} : _ -> isInExpansion t @@ -3839,7 +3840,7 @@ checkSubshelledTests params t = isFunctionBody path = case path of - (_:f:_) -> isFunction f + (_ NE.:| f:_) -> isFunction f _ -> False isTestStructure t = @@ -3866,7 +3867,7 @@ checkSubshelledTests params t = -- This technically also triggers for `if true; then ( test ); fi` -- but it's still a valid suggestion. isCompoundCondition chain = - case dropWhile skippable (drop 1 chain) of + case dropWhile skippable (NE.tail chain) of T_IfExpression {} : _ -> True T_WhileExpression {} : _ -> True T_UntilExpression {} : _ -> True @@ -4005,7 +4006,7 @@ checkUselessBang params t = when (hasSetE params) $ mapM_ check (getNonReturning where check t = case t of - T_Banged id cmd | not $ isCondition (getPath (parentMap params) t) -> + T_Banged id cmd | not $ isCondition (NE.toList $ getPath (parentMap params) t) -> addComment $ makeCommentWithFix InfoC id 2251 "This ! is not on a condition and skips errexit. Use `&& exit 1` instead, or make sure $? is checked." (fixWith [replaceStart id params 1 "", replaceEnd (getId cmd) params 0 " && exit 1"]) @@ -4029,7 +4030,7 @@ checkUselessBang params t = when (hasSetE params) $ mapM_ check (getNonReturning isFunctionBody t = case getPath (parentMap params) t of - _:T_Function {}:_-> True + _ NE.:| T_Function {}:_-> True _ -> False dropLast t = @@ -4627,7 +4628,7 @@ checkArrayValueUsedAsIndex params _ = -- Is this one of the 'for' arrays? (loopWord, _) <- find ((==arrayName) . snd) arrays -- Are we still in this loop? - guard $ getId loop `elem` map getId (getPath parents t) + guard $ getId loop `elem` NE.map getId (getPath parents t) return [ makeComment WarningC (getId loopWord) 2302 "This loops over values. To loop over keys, use \"${!array[@]}\".", makeComment WarningC (getId arrayRef) 2303 $ (e4m name) ++ " is an array value, not a key. Use directly or loop over keys instead." @@ -4709,7 +4710,7 @@ checkSetESuppressed params t = literalArg <- getUnquotedLiteral cmd Map.lookup literalArg functions_ - checkCmd cmd = go $ getPath (parentMap params) cmd + checkCmd cmd = go $ NE.toList $ getPath (parentMap params) cmd where go (child:parent:rest) = do case parent of @@ -4855,7 +4856,7 @@ checkExtraMaskedReturns params t = basename <- getCommandBasename t return $ basename == "time" - parentChildPairs t = go $ parents params t + parentChildPairs t = go $ NE.toList $ parents params t where go (child:parent:rest) = (parent, child):go (parent:rest) go _ = [] diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 1d53a98..21123d4 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -41,6 +41,7 @@ import Data.Char import Data.List import Data.Maybe import Data.Semigroup +import qualified Data.List.NonEmpty as NE import qualified Data.Map as Map import Test.QuickCheck.All (forAllProperties) @@ -336,7 +337,7 @@ isQuoteFree = isQuoteFreeNode False isQuoteFreeNode strict shell tree t = isQuoteFreeElement t || - (fromMaybe False $ msum $ map isQuoteFreeContext $ drop 1 $ getPath tree t) + (fromMaybe False $ msum $ map isQuoteFreeContext $ NE.tail $ getPath tree t) where -- Is this node self-quoting in itself? isQuoteFreeElement t = @@ -398,7 +399,7 @@ isParamTo tree cmd = -- Get the parent command (T_Redirecting) of a Token, if any. getClosestCommand :: Map.Map Id Token -> Token -> Maybe Token getClosestCommand tree t = - findFirst findCommand $ getPath tree t + findFirst findCommand $ NE.toList $ getPath tree t where findCommand t = case t of @@ -412,7 +413,7 @@ getClosestCommandM t = do return $ getClosestCommand (parentMap params) t -- Is the token used as a command name (the first word in a T_SimpleCommand)? -usedAsCommandName tree token = go (getId token) (tail $ getPath tree token) +usedAsCommandName tree token = go (getId token) (NE.tail $ getPath tree token) where go currentId (T_NormalWord id [word]:rest) | currentId == getId word = go id rest @@ -429,7 +430,7 @@ getPathM t = do return $ getPath (parentMap params) t isParentOf tree parent child = - elem (getId parent) . map getId $ getPath tree child + elem (getId parent) . NE.map getId $ getPath tree child parents params = getPath (parentMap params) @@ -813,7 +814,7 @@ getReferencedVariables parents t = return (context, token, getBracedReference str) isArithmeticAssignment t = case getPath parents t of - this: TA_Assignment _ "=" lhs _ :_ -> lhs == t + this NE.:| TA_Assignment _ "=" lhs _ :_ -> lhs == t _ -> False isDereferencingBinaryOp = (`elem` ["-eq", "-ne", "-lt", "-le", "-gt", "-ge"]) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 429e786..c4ffd87 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -43,6 +43,7 @@ import Data.Functor.Identity import qualified Data.Graph.Inductive.Graph as G import Data.List import Data.Maybe +import qualified Data.List.NonEmpty as NE import qualified Data.Map.Strict as M import qualified Data.Set as S import Test.QuickCheck.All (forAllProperties) @@ -1005,7 +1006,7 @@ checkWhileGetoptsCase = CommandCheck (Exactly "getopts") f sequence_ $ do options <- getLiteralString arg1 getoptsVar <- getLiteralString name - (T_WhileExpression _ _ body) <- findFirst whileLoop path + (T_WhileExpression _ _ body) <- findFirst whileLoop (NE.toList path) T_CaseExpression id var list <- mapMaybe findCase body !!! 0 -- Make sure getopts name and case variable matches From 71c0fcb737e94d7f2aa65e0540ddc2554f63bdd7 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 31 Dec 2023 02:27:52 -0500 Subject: [PATCH 361/459] Manually fuse elem and map in isParentOf --- src/ShellCheck/AnalyzerLib.hs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 21123d4..944b12d 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -430,7 +430,9 @@ getPathM t = do return $ getPath (parentMap params) t isParentOf tree parent child = - elem (getId parent) . NE.map getId $ getPath tree child + any (\t -> parentId == getId t) (getPath tree child) + where + parentId = getId parent parents params = getPath (parentMap params) From 6e5b5401c6e593c82ce11ab15b532fe3a9c07d3b Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 31 Dec 2023 02:31:07 -0500 Subject: [PATCH 362/459] Manually fuse elem and map in checkArrayValueUsedAsIndex --- src/ShellCheck/Analytics.hs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index e80ed58..9926462 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -4628,7 +4628,8 @@ checkArrayValueUsedAsIndex params _ = -- Is this one of the 'for' arrays? (loopWord, _) <- find ((==arrayName) . snd) arrays -- Are we still in this loop? - guard $ getId loop `elem` NE.map getId (getPath parents t) + let loopId = getId loop + guard $ any (\t -> loopId == getId t) (getPath parents t) return [ makeComment WarningC (getId loopWord) 2302 "This loops over values. To loop over keys, use \"${!array[@]}\".", makeComment WarningC (getId arrayRef) 2303 $ (e4m name) ++ " is an array value, not a key. Use directly or loop over keys instead." From a786f996a176555ca5c09866dcb785cf7fd9323d Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 31 Dec 2023 15:55:06 -0500 Subject: [PATCH 363/459] Replace !!! with pattern-matching where it's easy --- src/ShellCheck/Analytics.hs | 14 +++++--------- src/ShellCheck/Checks/Commands.hs | 3 +-- src/ShellCheck/Checks/ShellSupport.hs | 5 ++--- 3 files changed, 8 insertions(+), 14 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 9926462..2fb3185 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -468,9 +468,8 @@ checkAssignAteCommand _ (T_SimpleCommand id [T_Assignment _ _ _ _ assignmentTerm where isCommonCommand (Just s) = s `elem` commonCommands isCommonCommand _ = False - firstWordIsArg list = fromMaybe False $ do - head <- list !!! 0 - return $ isGlob head || isUnquotedFlag head + firstWordIsArg (head:_) = isGlob head || isUnquotedFlag head + firstWordIsArg [] = False checkAssignAteCommand _ _ = return () @@ -491,9 +490,7 @@ prop_checkWrongArit2 = verify checkWrongArithmeticAssignment "n=2; i=n*2" checkWrongArithmeticAssignment params (T_SimpleCommand id [T_Assignment _ _ _ _ val] []) = sequence_ $ do str <- getNormalString val - match <- matchRegex regex str - var <- match !!! 0 - op <- match !!! 1 + var:op:_ <- matchRegex regex str Map.lookup var references return . warn (getId val) 2100 $ "Use $((..)) for arithmetics, e.g. i=$((i " ++ op ++ " 2))" @@ -1460,9 +1457,8 @@ prop_checkForDecimals2 = verify checkForDecimals "foo[1.2]=bar" prop_checkForDecimals3 = verifyNot checkForDecimals "declare -A foo; foo[1.2]=bar" checkForDecimals params t@(TA_Expansion id _) = sequence_ $ do guard $ not (hasFloatingPoint params) - str <- getLiteralString t - first <- str !!! 0 - guard $ isDigit first && '.' `elem` str + first:rest <- getLiteralString t + guard $ isDigit first && '.' `elem` rest return $ err id 2079 "(( )) doesn't support decimals. Use bc or awk." checkForDecimals _ _ = return () diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index c4ffd87..97c9088 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -1237,8 +1237,7 @@ checkSudoArgs = CommandCheck (Basename "sudo") f where f t = sequence_ $ do opts <- parseOpts $ arguments t - let nonFlags = [x | ("",(x, _)) <- opts] - commandArg <- nonFlags !!! 0 + (_,(commandArg, _)) <- find (null . fst) opts command <- getLiteralString commandArg guard $ command `elem` builtins return $ warn (getId t) 2232 $ "Can't use sudo with builtins like " ++ command ++ ". Did you want sudo sh -c .. instead?" diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index d070497..cab0546 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -79,9 +79,8 @@ prop_checkForDecimals3 = verifyNot checkForDecimals "declare -A foo; foo[1.2]=ba checkForDecimals = ForShell [Sh, Dash, BusyboxSh, Bash] f where f t@(TA_Expansion id _) = sequence_ $ do - str <- getLiteralString t - first <- str !!! 0 - guard $ isDigit first && '.' `elem` str + first:rest <- getLiteralString t + guard $ isDigit first && '.' `elem` rest return $ err id 2079 "(( )) doesn't support decimals. Use bc or awk." f _ = return () From 10afe83ce32c575cace720b51516092a44d39cf2 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 31 Dec 2023 16:23:45 -0500 Subject: [PATCH 364/459] Use getLiteralStringDef instead of rebuilding it with fromJust --- src/ShellCheck/Analytics.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 2fb3185..d030812 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -4509,7 +4509,7 @@ prop_checkCommandWithTrailingSymbol9 = verifyNot checkCommandWithTrailingSymbol checkCommandWithTrailingSymbol _ t = case t of T_SimpleCommand _ _ (cmd:_) -> - let str = fromJust $ getLiteralStringExt (\_ -> Just "x") cmd + let str = getLiteralStringDef "x" cmd last = lastOrDefault 'x' str in case str of From 6c81505870d84747c7614deebc2770f38c3e4c29 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 31 Dec 2023 16:26:03 -0500 Subject: [PATCH 365/459] Use a pattern guard instead of fromJust in checkLoopKeywordScope --- src/ShellCheck/Analytics.hs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index d030812..6daf614 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -19,6 +19,7 @@ -} {-# LANGUAGE TemplateHaskell #-} {-# LANGUAGE FlexibleContexts #-} +{-# LANGUAGE PatternGuards #-} module ShellCheck.Analytics (checker, optionalChecks, ShellCheck.Analytics.runTests) where import ShellCheck.AST @@ -2749,18 +2750,17 @@ prop_checkLoopKeywordScope5 = verify checkLoopKeywordScope "if true; then break; prop_checkLoopKeywordScope6 = verify checkLoopKeywordScope "while true; do true | { break; }; done" prop_checkLoopKeywordScope7 = verifyNot checkLoopKeywordScope "#!/bin/ksh\nwhile true; do true | { break; }; done" checkLoopKeywordScope params t | - name `elem` map Just ["continue", "break"] = + Just name <- getCommandName t, name `elem` ["continue", "break"] = if not $ any isLoop path then if any isFunction $ take 1 path -- breaking at a source/function invocation is an abomination. Let's ignore it. - then err (getId t) 2104 $ "In functions, use return instead of " ++ fromJust name ++ "." - else err (getId t) 2105 $ fromJust name ++ " is only valid in loops." + then err (getId t) 2104 $ "In functions, use return instead of " ++ name ++ "." + else err (getId t) 2105 $ name ++ " is only valid in loops." else case map subshellType $ filter (not . isFunction) path of Just str:_ -> warn (getId t) 2106 $ "This only exits the subshell caused by the " ++ str ++ "." _ -> return () where - name = getCommandName t path = let p = getPath (parentMap params) t in NE.filter relevant p subshellType t = case leadType params t of NoneScope -> Nothing From 3f40b688eee018f49b0a2ed7d8805987eb4118cd Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 31 Dec 2023 16:33:34 -0500 Subject: [PATCH 366/459] Simplify getStringFromParsec --- src/ShellCheck/Parser.hs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 37a9b86..130d956 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -3456,9 +3456,8 @@ makeErrorFor parsecError = pos = errorPos parsecError getStringFromParsec errors = - case map f errors of - r -> unwords (take 1 $ catMaybes $ reverse r) ++ - " Fix any mentioned problems and try again." + headOrDefault "" (mapMaybe f $ reverse errors) ++ + " Fix any mentioned problems and try again." where f err = case err of From a6984cddb0fbead307dce4bd98a66f8225ea2888 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 31 Dec 2023 16:40:18 -0500 Subject: [PATCH 367/459] Switch then and else to remove a not --- src/ShellCheck/Analytics.hs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 6daf614..ae29762 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2751,15 +2751,15 @@ prop_checkLoopKeywordScope6 = verify checkLoopKeywordScope "while true; do true prop_checkLoopKeywordScope7 = verifyNot checkLoopKeywordScope "#!/bin/ksh\nwhile true; do true | { break; }; done" checkLoopKeywordScope params t | Just name <- getCommandName t, name `elem` ["continue", "break"] = - if not $ any isLoop path - then if any isFunction $ take 1 path - -- breaking at a source/function invocation is an abomination. Let's ignore it. - then err (getId t) 2104 $ "In functions, use return instead of " ++ name ++ "." - else err (getId t) 2105 $ name ++ " is only valid in loops." - else case map subshellType $ filter (not . isFunction) path of + if any isLoop path + then case map subshellType $ filter (not . isFunction) path of Just str:_ -> warn (getId t) 2106 $ "This only exits the subshell caused by the " ++ str ++ "." _ -> return () + else if any isFunction $ take 1 path + -- breaking at a source/function invocation is an abomination. Let's ignore it. + then err (getId t) 2104 $ "In functions, use return instead of " ++ name ++ "." + else err (getId t) 2105 $ name ++ " is only valid in loops." where path = let p = getPath (parentMap params) t in NE.filter relevant p subshellType t = case leadType params t of From 71889c139aac3ce3597349878c746c774b329882 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 31 Dec 2023 16:44:21 -0500 Subject: [PATCH 368/459] Use a case expression instead of any and take 1 --- src/ShellCheck/Analytics.hs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index ae29762..ca679bb 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2756,10 +2756,10 @@ checkLoopKeywordScope params t | Just str:_ -> warn (getId t) 2106 $ "This only exits the subshell caused by the " ++ str ++ "." _ -> return () - else if any isFunction $ take 1 path + else case path of -- breaking at a source/function invocation is an abomination. Let's ignore it. - then err (getId t) 2104 $ "In functions, use return instead of " ++ name ++ "." - else err (getId t) 2105 $ name ++ " is only valid in loops." + h:_ | isFunction h -> err (getId t) 2104 $ "In functions, use return instead of " ++ name ++ "." + _ -> err (getId t) 2105 $ name ++ " is only valid in loops." where path = let p = getPath (parentMap params) t in NE.filter relevant p subshellType t = case leadType params t of From 7b0589988fca3bf234d2d62e475f2e821ddb42c5 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 31 Dec 2023 17:21:50 -0500 Subject: [PATCH 369/459] Implement isCondition in terms of foldr --- src/ShellCheck/Analytics.hs | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index ca679bb..3f686ee 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -346,13 +346,11 @@ dist a b hasFloatingPoint params = shellType params == Ksh -- Checks whether the current parent path is part of a condition -isCondition [] = False -isCondition [_] = False -isCondition (child:parent:rest) = - case child of - T_BatsTest {} -> True -- count anything in a @test as conditional - _ -> getId child `elem` map getId (getConditionChildren parent) || isCondition (parent:rest) +isCondition (x NE.:| xs) = foldr go (const False) xs x where + go _ _ T_BatsTest{} = True -- count anything in a @test as conditional + go parent go_rest child = + getId child `elem` map getId (getConditionChildren parent) || go_rest parent getConditionChildren t = case t of T_AndIf _ left right -> [left] @@ -886,7 +884,7 @@ checkShorthandIf params x@(T_OrIf _ (T_AndIf id _ _) (T_Pipeline _ _ t)) name <- getCommandBasename t return $ name `elem` ["echo", "exit", "return", "printf"]) isOk _ = False - inCondition = isCondition $ NE.toList $ getPath (parentMap params) x + inCondition = isCondition $ getPath (parentMap params) x checkShorthandIf _ _ = return () @@ -3185,7 +3183,7 @@ checkUncheckedCdPushdPopd params root = | name `elem` ["cd", "pushd", "popd"] && not (isSafeDir t) && not (name `elem` ["pushd", "popd"] && ("n" `elem` map snd (getAllFlags t))) - && not (isCondition $ NE.toList $ getPath (parentMap params) t) = + && not (isCondition $ getPath (parentMap params) t) = warnWithFix (getId t) 2164 ("Use '" ++ name ++ " ... || exit' or '" ++ name ++ " ... || return' in case " ++ name ++ " fails.") (fixWith [replaceEnd (getId t) params 0 " || exit"]) @@ -4002,7 +4000,7 @@ checkUselessBang params t = when (hasSetE params) $ mapM_ check (getNonReturning where check t = case t of - T_Banged id cmd | not $ isCondition (NE.toList $ getPath (parentMap params) t) -> + T_Banged id cmd | not $ isCondition (getPath (parentMap params) t) -> addComment $ makeCommentWithFix InfoC id 2251 "This ! is not on a condition and skips errexit. Use `&& exit 1` instead, or make sure $? is checked." (fixWith [replaceStart id params 1 "", replaceEnd (getId cmd) params 0 " && exit 1"]) From b7f88ec4b72bae672f1b77e4af312990f17ab86b Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 31 Dec 2023 18:09:02 -0500 Subject: [PATCH 370/459] Stop building tuples that we never look at both sides of --- src/ShellCheck/Analytics.hs | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 3f686ee..9913b09 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -4819,15 +4819,15 @@ checkExtraMaskedReturns params t = ++ "separately to avoid masking its return value (or use '|| true' " ++ "to ignore).") - isMaskDeliberate t = hasParent isOrIf t + isMaskDeliberate t = any isOrIf $ NE.init $ parents params t where - isOrIf _ (T_OrIf _ _ (T_Pipeline _ _ [T_Redirecting _ _ cmd])) + isOrIf (T_OrIf _ _ (T_Pipeline _ _ [T_Redirecting _ _ cmd])) = getCommandBasename cmd `elem` [Just "true", Just ":"] - isOrIf _ _ = False + isOrIf _ = False - isCheckedElsewhere t = hasParent isDeclaringCommand t + isCheckedElsewhere t = any isDeclaringCommand $ NE.tail $ parents params t where - isDeclaringCommand t _ = fromMaybe False $ do + isDeclaringCommand t = fromMaybe False $ do cmd <- getCommand t basename <- getCommandBasename cmd return $ @@ -4851,13 +4851,6 @@ checkExtraMaskedReturns params t = basename <- getCommandBasename t return $ basename == "time" - parentChildPairs t = go $ NE.toList $ parents params t - where - go (child:parent:rest) = (parent, child):go (parent:rest) - go _ = [] - - hasParent pred t = any (uncurry pred) (parentChildPairs t) - -- hard error on negated command that is not last prop_checkBatsTestDoesNotUseNegation1 = verify checkBatsTestDoesNotUseNegation "#!/usr/bin/env/bats\n@test \"name\" { ! true; false; }" From 9e0fdbe431f6a9725a461b214c97f885782f0314 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 31 Dec 2023 18:13:32 -0500 Subject: [PATCH 371/459] Simplify isTransparentCommand --- src/ShellCheck/Analytics.hs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 9913b09..71caa62 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -4847,9 +4847,7 @@ checkExtraMaskedReturns params t = ,"shopt" ] - isTransparentCommand t = fromMaybe False $ do - basename <- getCommandBasename t - return $ basename == "time" + isTransparentCommand t = getCommandBasename t == Just "time" -- hard error on negated command that is not last From 5a6f4840adf584da60273c6b8145c19fd7e342a4 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Mon, 1 Jan 2024 14:16:50 -0500 Subject: [PATCH 372/459] Replace a few more occurrences of !!! with pattern matching --- src/ShellCheck/ASTLib.hs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index aadff05..9b151c2 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -858,8 +858,7 @@ getBracedModifier s = headOrDefault "" $ do -- Get the variables from indices like ["x", "y"] in ${var[x+y+1]} prop_getIndexReferences1 = getIndexReferences "var[x+y+1]" == ["x", "y"] getIndexReferences s = fromMaybe [] $ do - match <- matchRegex re s - index <- match !!! 0 + index:_ <- matchRegex re s return $ matchAllStrings variableNameRegex index where re = mkRegex "(\\[.*\\])" @@ -870,8 +869,7 @@ prop_getOffsetReferences3 = getOffsetReferences "[foo]:bar" == ["bar"] prop_getOffsetReferences4 = getOffsetReferences "[foo]:bar:baz" == ["bar", "baz"] getOffsetReferences mods = fromMaybe [] $ do -- if mods start with [, then drop until ] - match <- matchRegex re mods - offsets <- match !!! 1 + _:offsets:_ <- matchRegex re mods return $ matchAllStrings variableNameRegex offsets where re = mkRegex "^(\\[.+\\])? *:([^-=?+].*)" From 025cc5266ec1632d5f524644f0366237f604e7e8 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Mon, 1 Jan 2024 16:00:19 -0500 Subject: [PATCH 373/459] Simplify isUnquotedFlag --- src/ShellCheck/ASTLib.hs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 9b151c2..88089ea 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -158,9 +158,10 @@ isFlag token = _ -> False -- Is this token a flag where the - is unquoted? -isUnquotedFlag token = fromMaybe False $ do - str <- getLeadingUnquotedString token - return $ "-" `isPrefixOf` str +isUnquotedFlag token = + case getLeadingUnquotedString token of + Just ('-':_) -> True + _ -> False -- getGnuOpts "erd:u:" will parse a list of arguments tokens like `read` -- -re -d : -u 3 bar From 67abfe159e41e33a7ad17d9b5e130756cc447510 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Mon, 1 Jan 2024 19:04:26 -0500 Subject: [PATCH 374/459] Remove most of the partial head and tail functions from src/ShellCheck/CFG.hs --- src/ShellCheck/CFG.hs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/ShellCheck/CFG.hs b/src/ShellCheck/CFG.hs index 5d018c8..e1d3259 100644 --- a/src/ShellCheck/CFG.hs +++ b/src/ShellCheck/CFG.hs @@ -615,15 +615,15 @@ build t = do T_CaseExpression id t [] -> build t - T_CaseExpression id t list -> do + T_CaseExpression id t list@(hd:tl) -> do start <- newStructuralNode token <- build t - branches <- mapM buildBranch list + branches <- mapM buildBranch (hd NE.:| tl) end <- newStructuralNode - let neighbors = zip branches $ tail branches - let (_, firstCond, _) = head branches - let (_, lastCond, lastBody) = last branches + let neighbors = zip (NE.toList branches) $ NE.tail branches + let (_, firstCond, _) = NE.head branches + let (_, lastCond, lastBody) = NE.last branches linkRange start token linkRange token firstCond From ba86c6363c30a5dbefd0b8b9a7c5f4ab0478dc91 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Tue, 2 Jan 2024 14:46:07 -0500 Subject: [PATCH 375/459] Use maybe instead of fromMaybe and fmap --- src/ShellCheck/Analytics.hs | 2 +- src/ShellCheck/CFGAnalysis.hs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 71caa62..3af0455 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -5020,7 +5020,7 @@ checkPlusEqualsNumber params t = let unquotedLiteral = getUnquotedLiteral word isEmpty = unquotedLiteral == Just "" - isUnquotedNumber = not isEmpty && fromMaybe False (all isDigit <$> unquotedLiteral) + isUnquotedNumber = not isEmpty && maybe False (all isDigit) unquotedLiteral isNumericalVariableName = fromMaybe False $ do str <- unquotedLiteral CF.variableMayBeAssignedInteger state str diff --git a/src/ShellCheck/CFGAnalysis.hs b/src/ShellCheck/CFGAnalysis.hs index 16afa68..0b99c9f 100644 --- a/src/ShellCheck/CFGAnalysis.hs +++ b/src/ShellCheck/CFGAnalysis.hs @@ -829,7 +829,7 @@ lookupStack' functionOnly get dep def ctx key = do f (s:rest) = do -- Go up the stack until we find the value, and add -- a dependency on each state (including where it was found) - res <- fromMaybe (f rest) (return <$> get (stackState s) key) + res <- maybe (f rest) return (get (stackState s) key) modifySTRef (dependencies s) $ S.insert $ dep key res return res From 1bce426fcfd6ff176800bd70d7a298cf261d8512 Mon Sep 17 00:00:00 2001 From: Georg Pfuetzenreuter Date: Sun, 21 Jan 2024 02:16:35 +0100 Subject: [PATCH 376/459] Implement rcfile option This introduces the "--rcfile" argument which allows a specific shellcheckrc file to be passed. If specified and the given file exists, the default locations will not be searched and the specified file will be used. Signed-off-by: Georg Pfuetzenreuter --- shellcheck.1.md | 5 +++++ shellcheck.hs | 39 +++++++++++++++++++++++++++------------ 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/shellcheck.1.md b/shellcheck.1.md index 89f6d50..42a0429 100644 --- a/shellcheck.1.md +++ b/shellcheck.1.md @@ -71,6 +71,11 @@ not warn at all, as `ksh` supports decimals in arithmetic contexts. : Don't try to look for .shellcheckrc configuration files. +--rcfile\ RCFILE + +: Prefer the specified configuration file over searching for one + in the default locations. + **-o**\ *NAME1*[,*NAME2*...],\ **--enable=***NAME1*[,*NAME2*...] : Enable optional checks. The special name *all* enables all of them. diff --git a/shellcheck.hs b/shellcheck.hs index 00b699b..42554f3 100644 --- a/shellcheck.hs +++ b/shellcheck.hs @@ -76,7 +76,8 @@ data Options = Options { externalSources :: Bool, sourcePaths :: [FilePath], formatterOptions :: FormatterOptions, - minSeverity :: Severity + minSeverity :: Severity, + rcfile :: FilePath } defaultOptions = Options { @@ -86,7 +87,8 @@ defaultOptions = Options { formatterOptions = newFormatterOptions { foColorOption = ColorAuto }, - minSeverity = StyleC + minSeverity = StyleC, + rcfile = [] } usageHeader = "Usage: shellcheck [OPTIONS...] FILES..." @@ -107,6 +109,9 @@ options = [ (NoArg $ Flag "list-optional" "true") "List checks disabled by default", Option "" ["norc"] (NoArg $ Flag "norc" "true") "Don't look for .shellcheckrc files", + Option "" ["rcfile"] + (ReqArg (Flag "rcfile") "RCFILE") + "Prefer the specified configuration file over searching for one", Option "o" ["enable"] (ReqArg (Flag "enable") "check1,check2..") "List of optional checks to enable (or 'all')", @@ -367,6 +372,11 @@ parseOption flag options = } } + Flag "rcfile" str -> do + return options { + rcfile = str + } + Flag "enable" value -> let cs = checkSpec options in return options { checkSpec = cs { @@ -441,18 +451,23 @@ ioInterface options files = do fallback :: FilePath -> IOException -> IO FilePath fallback path _ = return path + -- Returns the name and contents of .shellcheckrc for the given file getConfig cache filename = do - path <- normalize filename - let dir = takeDirectory path - (previousPath, result) <- readIORef cache - if dir == previousPath - then return result - else do - paths <- getConfigPaths dir - result <- findConfig paths - writeIORef cache (dir, result) - return result + contents <- readConfig (rcfile options) + if isNothing contents + then do + path <- normalize filename + let dir = takeDirectory path + (previousPath, result) <- readIORef cache + if dir == previousPath + then return result + else do + paths <- getConfigPaths dir + result <- findConfig paths + writeIORef cache (dir, result) + return result + else return contents findConfig paths = case paths of From de95624d310622d234149e5b1bc96da2a40ff317 Mon Sep 17 00:00:00 2001 From: Grische <2787581+grische@users.noreply.github.com> Date: Fri, 2 Feb 2024 12:28:09 +0100 Subject: [PATCH 377/459] Remove deprecated "install --enable-tests" command --- README.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/README.md b/README.md index ca9b847..3692319 100644 --- a/README.md +++ b/README.md @@ -309,10 +309,6 @@ Verify that `cabal` is installed and update its dependency list with $ cabal install -Or if you intend to run the tests: - - $ cabal install --enable-tests - This will compile ShellCheck and install it to your `~/.cabal/bin` directory. Add this directory to your `PATH` (for bash, add this to your `~/.bashrc`): @@ -558,4 +554,3 @@ Happy ShellChecking! * The wiki has [long form descriptions](https://github.com/koalaman/shellcheck/wiki/Checks) for each warning, e.g. [SC2221](https://github.com/koalaman/shellcheck/wiki/SC2221). * ShellCheck does not attempt to enforce any kind of formatting or indenting style, so also check out [shfmt](https://github.com/mvdan/sh)! - From 6a44a19f17c4a0590693587af3b5209d7b1b59fe Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 3 Feb 2024 13:34:49 -0800 Subject: [PATCH 378/459] Only read --rcfile once, and skip search if unavailable --- shellcheck.hs | 48 +++++++++++++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/shellcheck.hs b/shellcheck.hs index 42554f3..e933d6c 100644 --- a/shellcheck.hs +++ b/shellcheck.hs @@ -77,7 +77,7 @@ data Options = Options { sourcePaths :: [FilePath], formatterOptions :: FormatterOptions, minSeverity :: Severity, - rcfile :: FilePath + rcfile :: Maybe FilePath } defaultOptions = Options { @@ -88,7 +88,7 @@ defaultOptions = Options { foColorOption = ColorAuto }, minSeverity = StyleC, - rcfile = [] + rcfile = Nothing } usageHeader = "Usage: shellcheck [OPTIONS...] FILES..." @@ -374,7 +374,7 @@ parseOption flag options = Flag "rcfile" str -> do return options { - rcfile = str + rcfile = Just str } Flag "enable" value -> @@ -453,21 +453,31 @@ ioInterface options files = do -- Returns the name and contents of .shellcheckrc for the given file - getConfig cache filename = do - contents <- readConfig (rcfile options) - if isNothing contents - then do - path <- normalize filename - let dir = takeDirectory path - (previousPath, result) <- readIORef cache - if dir == previousPath - then return result - else do - paths <- getConfigPaths dir - result <- findConfig paths - writeIORef cache (dir, result) - return result - else return contents + getConfig cache filename = + case rcfile options of + Just file -> do + -- We have a specified rcfile. Ignore normal rcfile resolution. + (path, result) <- readIORef cache + if path == "/" + then return result + else do + result <- readConfig file + when (isNothing result) $ + hPutStrLn stderr $ "Warning: unable to read --rcfile " ++ file + writeIORef cache ("/", result) + return result + + Nothing -> do + path <- normalize filename + let dir = takeDirectory path + (previousPath, result) <- readIORef cache + if dir == previousPath + then return result + else do + paths <- getConfigPaths dir + result <- findConfig paths + writeIORef cache (dir, result) + return result findConfig paths = case paths of @@ -505,7 +515,7 @@ ioInterface options files = do where handler :: FilePath -> IOException -> IO (String, Bool) handler file err = do - putStrLn $ file ++ ": " ++ show err + hPutStrLn stderr $ file ++ ": " ++ show err return ("", True) andM a b arg = do From d80fdfa9e8e738827a88505b26d3e596c0f0e875 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 3 Feb 2024 15:45:23 -0800 Subject: [PATCH 379/459] Add extended-analysis directive to toggle DFA --- CHANGELOG.md | 5 +++- shellcheck.1.md | 13 +++++++++ shellcheck.hs | 18 +++++++++++++ src/ShellCheck/AST.hs | 1 + src/ShellCheck/ASTLib.hs | 6 +++++ src/ShellCheck/Analytics.hs | 21 +++++++++------ src/ShellCheck/AnalyzerLib.hs | 8 ++++-- src/ShellCheck/Checker.hs | 40 ++++++++++++++++++++++++++++ src/ShellCheck/Checks/Commands.hs | 18 +++++++------ src/ShellCheck/Checks/ControlFlow.hs | 2 +- src/ShellCheck/Interface.hs | 8 ++++-- src/ShellCheck/Parser.hs | 10 +++++++ 12 files changed, 128 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 897aa27..dc7f6ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ ## Git ### Added +- Added support for busybox sh +- Added flag --rcfile to specify an rc file by name. +- Added `extended-analysis=true` directive to enable/disable dataflow analysis + (with a corresponding --extended-analysis flag). - SC2324: Warn when x+=1 appends instead of increments - SC2325: Warn about multiple `!`s in dash/sh. - SC2326: Warn about `foo | ! bar` in bash/dash/sh. @@ -9,7 +13,6 @@ - SC3015: Warn bashism `test _ =~ _` like in [ ] - SC3016: Warn bashism `test -v _` like in [ ] - SC3017: Warn bashism `test -a _` like in [ ] -- Added support for busybox sh ### Fixed - source statements with here docs now work correctly diff --git a/shellcheck.1.md b/shellcheck.1.md index 42a0429..b2bef3c 100644 --- a/shellcheck.1.md +++ b/shellcheck.1.md @@ -56,6 +56,13 @@ not warn at all, as `ksh` supports decimals in arithmetic contexts. options are cumulative, but all the codes can be specified at once, comma-separated as a single argument. +**--extended-analysis=true/false** + +: Enable/disable Dataflow Analysis to identify more issues (default true). If + ShellCheck uses too much CPU/RAM when checking scripts with several + thousand lines of code, extended analysis can be disabled with this flag + or a directive. This flag overrides directives and rc files. + **-f** *FORMAT*, **--format=***FORMAT* : Specify the output format of shellcheck, which prints its results in the @@ -249,6 +256,12 @@ Valid keys are: : Enable an optional check by name, as listed with **--list-optional**. Only file-wide `enable` directives are considered. +**extended-analysis** +: Set to true/false to enable/disable dataflow analysis. Specifying + `# shellcheck extended-analysis=false` in particularly large (2000+ line) + auto-generated scripts will reduce ShellCheck's resource usage at the + expense of certain checks. Extended analysis is enabled by default. + **external-sources** : Set to `true` in `.shellcheckrc` to always allow ShellCheck to open arbitrary files from 'source' statements (the way most tools do). diff --git a/shellcheck.hs b/shellcheck.hs index e933d6c..def3654 100644 --- a/shellcheck.hs +++ b/shellcheck.hs @@ -102,6 +102,8 @@ options = [ (ReqArg (Flag "include") "CODE1,CODE2..") "Consider only given types of warnings", Option "e" ["exclude"] (ReqArg (Flag "exclude") "CODE1,CODE2..") "Exclude types of warnings", + Option "" ["extended-analysis"] + (ReqArg (Flag "extended-analysis") "bool") "Perform dataflow analysis (default true)", Option "f" ["format"] (ReqArg (Flag "format") "FORMAT") $ "Output format (" ++ formatList ++ ")", @@ -384,6 +386,14 @@ parseOption flag options = } } + Flag "extended-analysis" str -> do + value <- parseBool str + return options { + checkSpec = (checkSpec options) { + csExtendedAnalysis = Just value + } + } + -- This flag is handled specially in 'process' Flag "format" _ -> return options @@ -401,6 +411,14 @@ parseOption flag options = throwError SyntaxFailure return (Prelude.read num :: Integer) + parseBool str = do + case str of + "true" -> return True + "false" -> return False + _ -> do + printErr $ "Invalid boolean, expected true/false: " ++ str + throwError SyntaxFailure + ioInterface :: Options -> [FilePath] -> IO (SystemInterface IO) ioInterface options files = do inputs <- mapM normalize files diff --git a/src/ShellCheck/AST.hs b/src/ShellCheck/AST.hs index 5c20416..ca05c98 100644 --- a/src/ShellCheck/AST.hs +++ b/src/ShellCheck/AST.hs @@ -152,6 +152,7 @@ data Annotation = | ShellOverride String | SourcePath String | ExternalSources Bool + | ExtendedAnalysis Bool deriving (Show, Eq) data ConditionType = DoubleBracket | SingleBracket deriving (Show, Eq) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 88089ea..6b26b22 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -910,5 +910,11 @@ getEnableDirectives root = T_Annotation _ list _ -> [s | EnableComment s <- list] _ -> [] +getExtendedAnalysisDirective :: Token -> Maybe Bool +getExtendedAnalysisDirective root = + case root of + T_Annotation _ list _ -> listToMaybe $ [s | ExtendedAnalysis s <- list] + _ -> Nothing + return [] runTests = $quickCheckAll diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 3af0455..f885842 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1262,7 +1262,8 @@ checkNumberComparisons params (TC_Binary id typ op lhs rhs) = do str = concat $ oversimplify c var = getBracedReference str in fromMaybe False $ do - state <- CF.getIncomingState (cfgAnalysis params) id + cfga <- cfgAnalysis params + state <- CF.getIncomingState cfga id value <- Map.lookup var $ CF.variablesInScope state return $ CF.numericalStatus (CF.variableValue value) >= CF.NumericalStatusMaybe _ -> @@ -2143,7 +2144,8 @@ checkSpacefulnessCfg' dirtyPass params token@(T_DollarBraced id _ list) = && not (usedAsCommandName parents token) isClean = fromMaybe False $ do - state <- CF.getIncomingState (cfgAnalysis params) id + cfga <- cfgAnalysis params + state <- CF.getIncomingState cfga id value <- Map.lookup name $ CF.variablesInScope state return $ isCleanState value @@ -4896,7 +4898,8 @@ prop_checkCommandIsUnreachable3 = verifyNot checkCommandIsUnreachable "foo; bar checkCommandIsUnreachable params t = case t of T_Pipeline {} -> sequence_ $ do - state <- CF.getIncomingState (cfgAnalysis params) id + cfga <- cfgAnalysis params + state <- CF.getIncomingState cfga id guard . not $ CF.stateIsReachable state guard . not $ isSourced params t return $ info id 2317 "Command appears to be unreachable. Check usage (or ignore if invoked indirectly)." @@ -4918,14 +4921,15 @@ checkOverwrittenExitCode params t = _ -> return () where check id = sequence_ $ do - state <- CF.getIncomingState (cfgAnalysis params) id + cfga <- cfgAnalysis params + state <- CF.getIncomingState cfga id let exitCodeIds = CF.exitCodes state guard . not $ S.null exitCodeIds let idToToken = idMap params exitCodeTokens <- traverse (\k -> Map.lookup k idToToken) $ S.toList exitCodeIds return $ do - when (all isCondition exitCodeTokens && not (usedUnconditionally t exitCodeIds)) $ + when (all isCondition exitCodeTokens && not (usedUnconditionally cfga t exitCodeIds)) $ warn id 2319 "This $? refers to a condition, not a command. Assign to a variable to avoid it being overwritten." when (all isPrinting exitCodeTokens) $ warn id 2320 "This $? refers to echo/printf, not a previous command. Assign to variable to avoid it being overwritten." @@ -4938,8 +4942,8 @@ checkOverwrittenExitCode params t = -- If we don't do anything based on the condition, assume we wanted the condition itself -- This helps differentiate `x; [ $? -gt 0 ] && exit $?` vs `[ cond ]; exit $?` - usedUnconditionally t testIds = - all (\c -> CF.doesPostDominate (cfgAnalysis params) (getId t) c) testIds + usedUnconditionally cfga t testIds = + all (\c -> CF.doesPostDominate cfga (getId t) c) testIds isPrinting t = case getCommandBasename t of @@ -5009,7 +5013,8 @@ prop_checkPlusEqualsNumber9 = verifyNot checkPlusEqualsNumber "declare -ia var; checkPlusEqualsNumber params t = case t of T_Assignment id Append var _ word -> sequence_ $ do - state <- CF.getIncomingState (cfgAnalysis params) id + cfga <- cfgAnalysis params + state <- CF.getIncomingState cfga id guard $ isNumber state word guard . not $ fromMaybe False $ CF.variableMayBeDeclaredInteger state var return $ warn id 2324 "var+=1 will append, not increment. Use (( var += 1 )), declare -i var, or quote number to silence." diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 944b12d..d265ace 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -104,7 +104,7 @@ data Parameters = Parameters { -- map from token id to start and end position tokenPositions :: Map.Map Id (Position, Position), -- Result from Control Flow Graph analysis (including data flow analysis) - cfgAnalysis :: CF.CFGAnalysis + cfgAnalysis :: Maybe CF.CFGAnalysis } deriving (Show) -- TODO: Cache results of common AST ops here @@ -197,8 +197,10 @@ makeCommentWithFix severity id code str fix = } in force withFix +-- makeParameters :: CheckSpec -> Parameters makeParameters spec = params where + extendedAnalysis = fromMaybe True $ msum [asExtendedAnalysis spec, getExtendedAnalysisDirective root] params = Parameters { rootNode = root, shellType = fromMaybe (determineShell (asFallbackShell spec) root) $ asShellType spec, @@ -229,7 +231,9 @@ makeParameters spec = params parentMap = getParentTree root, variableFlow = getVariableFlow params root, tokenPositions = asTokenPositions spec, - cfgAnalysis = CF.analyzeControlFlow cfParams root + cfgAnalysis = do + guard extendedAnalysis + return $ CF.analyzeControlFlow cfParams root } cfParams = CF.CFGParameters { CF.cfLastpipe = hasLastpipe params, diff --git a/src/ShellCheck/Checker.hs b/src/ShellCheck/Checker.hs index 6c9166f..0cfc3ab 100644 --- a/src/ShellCheck/Checker.hs +++ b/src/ShellCheck/Checker.hs @@ -25,6 +25,7 @@ import ShellCheck.ASTLib import ShellCheck.Interface import ShellCheck.Parser +import Debug.Trace -- DO NOT SUBMIT import Data.Either import Data.Functor import Data.List @@ -86,6 +87,7 @@ checkScript sys spec = do asCheckSourced = csCheckSourced spec, asExecutionMode = Executed, asTokenPositions = tokenPositions, + asExtendedAnalysis = csExtendedAnalysis spec, asOptionalChecks = getEnableDirectives root ++ csOptionalChecks spec } where as = newAnalysisSpec root let analysisMessages = @@ -520,5 +522,43 @@ prop_hereDocsWillHaveParsedIndices = null result where result = check "#!/bin/bash\nmy_array=(a b)\ncat <> ./test\n $(( 1 + my_array[1] ))\nEOF" +prop_rcCanSuppressDfa = null result + where + result = checkWithRc "extended-analysis=false" emptyCheckSpec { + csScript = "#!/bin/sh\nexit; foo;" + } + +prop_fileCanSuppressDfa = null $ traceShowId result + where + result = checkWithRc "" emptyCheckSpec { + csScript = "#!/bin/sh\n# shellcheck extended-analysis=false\nexit; foo;" + } + +prop_fileWinsWhenSuppressingDfa1 = null result + where + result = checkWithRc "extended-analysis=true" emptyCheckSpec { + csScript = "#!/bin/sh\n# shellcheck extended-analysis=false\nexit; foo;" + } + +prop_fileWinsWhenSuppressingDfa2 = result == [2317] + where + result = checkWithRc "extended-analysis=false" emptyCheckSpec { + csScript = "#!/bin/sh\n# shellcheck extended-analysis=true\nexit; foo;" + } + +prop_flagWinsWhenSuppressingDfa1 = result == [2317] + where + result = checkWithRc "extended-analysis=false" emptyCheckSpec { + csScript = "#!/bin/sh\n# shellcheck extended-analysis=false\nexit; foo;", + csExtendedAnalysis = Just True + } + +prop_flagWinsWhenSuppressingDfa2 = null result + where + result = checkWithRc "extended-analysis=true" emptyCheckSpec { + csScript = "#!/bin/sh\n# shellcheck extended-analysis=true\nexit; foo;", + csExtendedAnalysis = Just False + } + return [] runTests = $quickCheckAll diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 97c9088..c10016e 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -1430,26 +1430,28 @@ prop_checkBackreferencingDeclaration6 = verify (checkBackreferencingDeclaration prop_checkBackreferencingDeclaration7 = verify (checkBackreferencingDeclaration "declare") "declare x=var $k=$x" checkBackreferencingDeclaration cmd = CommandCheck (Exactly cmd) check where - check t = foldM_ perArg M.empty $ arguments t + check t = do + cfga <- asks cfgAnalysis + when (isJust cfga) $ + foldM_ (perArg $ fromJust cfga) M.empty $ arguments t - perArg leftArgs t = + perArg cfga leftArgs t = case t of T_Assignment id _ name idx t -> do - warnIfBackreferencing leftArgs $ t:idx + warnIfBackreferencing cfga leftArgs $ t:idx return $ M.insert name id leftArgs t -> do - warnIfBackreferencing leftArgs [t] + warnIfBackreferencing cfga leftArgs [t] return leftArgs - warnIfBackreferencing backrefs l = do - references <- findReferences l + warnIfBackreferencing cfga backrefs l = do + references <- findReferences cfga l let reused = M.intersection backrefs references mapM msg $ M.toList reused msg (name, id) = warn id 2318 $ "This assignment is used again in this '" ++ cmd ++ "', but won't have taken effect. Use two '" ++ cmd ++ "'s." - findReferences list = do - cfga <- asks cfgAnalysis + findReferences cfga list = do let graph = CF.graph cfga let nodesMap = CF.tokenToNodes cfga let nodes = S.unions $ map (\id -> M.findWithDefault S.empty id nodesMap) $ map getId $ list diff --git a/src/ShellCheck/Checks/ControlFlow.hs b/src/ShellCheck/Checks/ControlFlow.hs index 9b7635e..d23fa15 100644 --- a/src/ShellCheck/Checks/ControlFlow.hs +++ b/src/ShellCheck/Checks/ControlFlow.hs @@ -78,7 +78,7 @@ controlFlowEffectChecks = [ runNodeChecks :: [ControlFlowNodeCheck] -> ControlFlowCheck runNodeChecks perNode = do cfg <- asks cfgAnalysis - runOnAll cfg + sequence_ $ runOnAll <$> cfg where getData datas n@(node, label) = do (pre, post) <- M.lookup node datas diff --git a/src/ShellCheck/Interface.hs b/src/ShellCheck/Interface.hs index c574cee..04e3c5a 100644 --- a/src/ShellCheck/Interface.hs +++ b/src/ShellCheck/Interface.hs @@ -21,11 +21,11 @@ module ShellCheck.Interface ( SystemInterface(..) - , CheckSpec(csFilename, csScript, csCheckSourced, csIncludedWarnings, csExcludedWarnings, csShellTypeOverride, csMinSeverity, csIgnoreRC, csOptionalChecks) + , CheckSpec(csFilename, csScript, csCheckSourced, csIncludedWarnings, csExcludedWarnings, csShellTypeOverride, csMinSeverity, csIgnoreRC, csExtendedAnalysis, csOptionalChecks) , CheckResult(crFilename, crComments) , ParseSpec(psFilename, psScript, psCheckSourced, psIgnoreRC, psShellTypeOverride) , ParseResult(prComments, prTokenPositions, prRoot) - , AnalysisSpec(asScript, asShellType, asFallbackShell, asExecutionMode, asCheckSourced, asTokenPositions, asOptionalChecks) + , AnalysisSpec(asScript, asShellType, asFallbackShell, asExecutionMode, asCheckSourced, asTokenPositions, asExtendedAnalysis, asOptionalChecks) , AnalysisResult(arComments) , FormatterOptions(foColorOption, foWikiLinkCount) , Shell(Ksh, Sh, Bash, Dash, BusyboxSh) @@ -100,6 +100,7 @@ data CheckSpec = CheckSpec { csIncludedWarnings :: Maybe [Integer], csShellTypeOverride :: Maybe Shell, csMinSeverity :: Severity, + csExtendedAnalysis :: Maybe Bool, csOptionalChecks :: [String] } deriving (Show, Eq) @@ -124,6 +125,7 @@ emptyCheckSpec = CheckSpec { csIncludedWarnings = Nothing, csShellTypeOverride = Nothing, csMinSeverity = StyleC, + csExtendedAnalysis = Nothing, csOptionalChecks = [] } @@ -174,6 +176,7 @@ data AnalysisSpec = AnalysisSpec { asExecutionMode :: ExecutionMode, asCheckSourced :: Bool, asOptionalChecks :: [String], + asExtendedAnalysis :: Maybe Bool, asTokenPositions :: Map.Map Id (Position, Position) } @@ -184,6 +187,7 @@ newAnalysisSpec token = AnalysisSpec { asExecutionMode = Executed, asCheckSourced = False, asOptionalChecks = [], + asExtendedAnalysis = Nothing, asTokenPositions = Map.empty } diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 130d956..9cc5e02 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -1058,6 +1058,16 @@ readAnnotationWithoutPrefix sandboxed = do "This shell type is unknown. Use e.g. sh or bash." return [ShellOverride shell] + "extended-analysis" -> do + pos <- getPosition + value <- plainOrQuoted $ many1 letter + case value of + "true" -> return [ExtendedAnalysis True] + "false" -> return [ExtendedAnalysis False] + _ -> do + parseNoteAt pos ErrorC 1146 "Unknown extended-analysis value. Expected true/false." + return [] + "external-sources" -> do pos <- getPosition value <- plainOrQuoted $ many1 letter From 8c4c112c2504b630b5e58a87c8f42bbe9c955946 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 19 Feb 2024 09:28:04 -0800 Subject: [PATCH 380/459] Initial version of an ARM64 macOS build --- .github/workflows/build.yml | 2 +- build/darwin.aarch64/Dockerfile | 40 +++++++++++++++++++++++++++++++++ build/darwin.aarch64/build | 15 +++++++++++++ build/darwin.aarch64/tag | 1 + 4 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 build/darwin.aarch64/Dockerfile create mode 100755 build/darwin.aarch64/build create mode 100644 build/darwin.aarch64/tag diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3e6fb27..c49b25a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -47,7 +47,7 @@ jobs: needs: package_source strategy: matrix: - build: [linux.x86_64, linux.aarch64, linux.armv6hf, darwin.x86_64, windows.x86_64] + build: [linux.x86_64, linux.aarch64, linux.armv6hf, darwin.x86_64, darwin.aarch64, windows.x86_64] runs-on: ubuntu-latest steps: - name: Checkout repository diff --git a/build/darwin.aarch64/Dockerfile b/build/darwin.aarch64/Dockerfile new file mode 100644 index 0000000..7839728 --- /dev/null +++ b/build/darwin.aarch64/Dockerfile @@ -0,0 +1,40 @@ +FROM ghcr.io/shepherdjerred/macos-cross-compiler:latest + +ENV TARGET aarch64-apple-darwin22 +ENV TARGETNAME darwin.aarch64 + +# Build dependencies +USER root +ENV DEBIAN_FRONTEND noninteractive +ENV LC_ALL C.utf8 + +# Install basic deps +RUN apt-get update && apt-get install -y automake autoconf build-essential curl xz-utils qemu-user-static + +# Install a more suitable host compiler +WORKDIR /host-ghc +RUN curl -L "https://downloads.haskell.org/~cabal/cabal-install-3.9.0.0/cabal-install-3.9-x86_64-linux-alpine.tar.xz" | tar xJv -C /usr/local/bin +RUN curl -L 'https://downloads.haskell.org/~ghc/8.10.7/ghc-8.10.7-x86_64-deb10-linux.tar.xz' | tar xJ --strip-components=1 +RUN ./configure && make install + +# Build GHC. We have to use an old version because cross-compilation across OS has since broken. +WORKDIR /ghc +RUN curl -L "https://downloads.haskell.org/~ghc/8.10.7/ghc-8.10.7-src.tar.xz" | tar xJ --strip-components=1 +RUN apt-get install -y llvm-12 +RUN ./boot && ./configure --host x86_64-linux-gnu --build x86_64-linux-gnu --target "$TARGET" +RUN cp mk/flavours/quick-cross.mk mk/build.mk && make -j "$(nproc)" +RUN make install + +# Due to an apparent cabal bug, we specify our options directly to cabal +# It won't reuse caches if ghc-options are specified in ~/.cabal/config +ENV CABALOPTS "--ghc-options;-optc-Os -optc-fPIC;--with-ghc=$TARGET-ghc;--with-hc-pkg=$TARGET-ghc-pkg;--constraint=hashable==1.3.5.0" + +# Prebuild the dependencies +RUN cabal update +RUN IFS=';' && cabal install --dependencies-only $CABALOPTS ShellCheck + +# Copy the build script +COPY build /usr/bin + +WORKDIR /scratch +ENTRYPOINT ["/usr/bin/build"] diff --git a/build/darwin.aarch64/build b/build/darwin.aarch64/build new file mode 100755 index 0000000..c15717a --- /dev/null +++ b/build/darwin.aarch64/build @@ -0,0 +1,15 @@ +#!/bin/sh +set -xe +{ + tar xzv --strip-components=1 + chmod +x striptests && ./striptests + mkdir "$TARGETNAME" + cabal update + ( IFS=';'; cabal build $CABALOPTS ) + find . -name shellcheck -type f -exec mv {} "$TARGETNAME/" \; + ls -l "$TARGETNAME" + "$TARGET-strip" "$TARGETNAME/shellcheck" + ls -l "$TARGETNAME" + file "$TARGETNAME/shellcheck" | grep "Mach-O 64-bit arm64 executable" +} >&2 +tar czv "$TARGETNAME" diff --git a/build/darwin.aarch64/tag b/build/darwin.aarch64/tag new file mode 100644 index 0000000..ae93ef3 --- /dev/null +++ b/build/darwin.aarch64/tag @@ -0,0 +1 @@ +koalaman/scbuilder-darwin-aarch64 From 55be4543f225824e1f6534973ad51c7833343e4e Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 19 Feb 2024 11:40:30 -0800 Subject: [PATCH 381/459] Avoid stripping darwin.aarch64 binaries to keep code signature --- build/darwin.aarch64/build | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build/darwin.aarch64/build b/build/darwin.aarch64/build index c15717a..4235d3a 100755 --- a/build/darwin.aarch64/build +++ b/build/darwin.aarch64/build @@ -8,7 +8,9 @@ set -xe ( IFS=';'; cabal build $CABALOPTS ) find . -name shellcheck -type f -exec mv {} "$TARGETNAME/" \; ls -l "$TARGETNAME" - "$TARGET-strip" "$TARGETNAME/shellcheck" + # Stripping invalidates the code signature and the build image does + # not appear to have anything similar to the 'codesign' tool. + # "$TARGET-strip" "$TARGETNAME/shellcheck" ls -l "$TARGETNAME" file "$TARGETNAME/shellcheck" | grep "Mach-O 64-bit arm64 executable" } >&2 From ad3c3146f0e10fcd331a1c8bba29e710f4417c99 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 3 Mar 2024 12:34:29 -0800 Subject: [PATCH 382/459] Fix snap build --- .snapsquid.conf | 14 -------------- snap/snapcraft.yaml | 18 +++++++++--------- 2 files changed, 9 insertions(+), 23 deletions(-) delete mode 100644 .snapsquid.conf diff --git a/.snapsquid.conf b/.snapsquid.conf deleted file mode 100644 index 205c1a6..0000000 --- a/.snapsquid.conf +++ /dev/null @@ -1,14 +0,0 @@ -# In 2015, cabal-install had a http bug triggered when proxies didn't keep -# the connection open. This version made it into Ubuntu Xenial as used by -# Snapcraft. In June 2018, Snapcraft's proxy started triggering this bug. -# -# https://bugs.launchpad.net/launchpad-buildd/+bug/1797809 -# -# Workaround: add more proxy - -visible_hostname localhost -http_port 8888 -cache_peer 10.10.10.1 parent 8222 0 no-query default -cache_peer_domain localhost !.internal -http_access allow all - diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index e14b854..f294c4e 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -23,7 +23,7 @@ description: | # snap connect shellcheck:removable-media version: git -base: core18 +base: core20 grade: stable confinement: strict @@ -40,16 +40,16 @@ parts: source: . build-packages: - cabal-install - - squid + stage-packages: + - libatomic1 override-build: | - # See comments in .snapsquid.conf - [ "$http_proxy" ] && { - squid3 -f .snapsquid.conf - export http_proxy="http://localhost:8888" - sleep 3 - } + # Give ourselves enough memory to build + dd if=/dev/zero of=/tmp/swap bs=1M count=2000 + mkswap /tmp/swap + swapon /tmp/swap + cabal sandbox init - cabal update || cat /var/log/squid/* + cabal update cabal install -j install -d $SNAPCRAFT_PART_INSTALL/usr/bin From 8bc7345aa7ec55f79a433388f443dddf4e89e270 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 3 Mar 2024 16:11:44 -0800 Subject: [PATCH 383/459] Remove outdated distros from testing --- test/distrotest | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/distrotest b/test/distrotest index 4ad66f8..ef467b8 100755 --- a/test/distrotest +++ b/test/distrotest @@ -76,8 +76,6 @@ archlinux:latest pacman -S -y --noconfirm cabal-install ghc-static base-dev # Ubuntu LTS ubuntu:22.04 apt-get update && apt-get install -y cabal-install ubuntu:20.04 apt-get update && apt-get install -y cabal-install -ubuntu:18.04 apt-get update && apt-get install -y cabal-install -ubuntu:16.04 apt-get update && apt-get install -y cabal-install # Stack on Ubuntu LTS ubuntu:22.04 set -e; apt-get update && apt-get install -y curl && curl -sSL https://get.haskellstack.org/ | sh -s - -f && cd /mnt && exec test/stacktest From a7e65dca8d7b0d19db9808e9ae17e2aa86ddbba4 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 4 Mar 2024 09:19:51 -0800 Subject: [PATCH 384/459] Update some copyright years --- shellcheck.1.md | 2 +- src/ShellCheck/Analytics.hs | 2 +- src/ShellCheck/Interface.hs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/shellcheck.1.md b/shellcheck.1.md index b2bef3c..b873e45 100644 --- a/shellcheck.1.md +++ b/shellcheck.1.md @@ -397,7 +397,7 @@ long list of wonderful contributors. # COPYRIGHT -Copyright 2012-2022, Vidar Holen and contributors. +Copyright 2012-2024, Vidar Holen and contributors. Licensed under the GNU General Public License version 3 or later, see https://gnu.org/licenses/gpl.html diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index f885842..1cc8bf8 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1,5 +1,5 @@ {- - Copyright 2012-2022 Vidar Holen + Copyright 2012-2024 Vidar Holen This file is part of ShellCheck. https://www.shellcheck.net diff --git a/src/ShellCheck/Interface.hs b/src/ShellCheck/Interface.hs index 04e3c5a..16a7e36 100644 --- a/src/ShellCheck/Interface.hs +++ b/src/ShellCheck/Interface.hs @@ -1,5 +1,5 @@ {- - Copyright 2012-2019 Vidar Holen + Copyright 2012-2024 Vidar Holen This file is part of ShellCheck. https://www.shellcheck.net From 37dfb67768db726092fde482d338943d678e6988 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 7 Mar 2024 17:53:15 -0800 Subject: [PATCH 385/459] Stable version v0.10.0 This release is dedicated to LLMs, for finally fulfilling the promise of 1960s scifi: systems you can hack using logic games and creative lies. --- CHANGELOG.md | 6 +++--- ShellCheck.cabal | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dc7f6ea..6c8beeb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ -## Git +## v0.10.0 - 2024-03-07 ### Added +- Precompiled binaries for macOS ARM64 (darwin.aarch64) - Added support for busybox sh - Added flag --rcfile to specify an rc file by name. - Added `extended-analysis=true` directive to enable/disable dataflow analysis @@ -16,8 +17,7 @@ ### Fixed - source statements with here docs now work correctly - -### Changed +- "(Array.!): undefined array element" error should no longer occur ## v0.9.0 - 2022-12-12 diff --git a/ShellCheck.cabal b/ShellCheck.cabal index a12f75e..fc52b12 100644 --- a/ShellCheck.cabal +++ b/ShellCheck.cabal @@ -1,5 +1,5 @@ Name: ShellCheck -Version: 0.9.0 +Version: 0.10.0 Synopsis: Shell script analysis tool License: GPL-3 License-file: LICENSE From 94214ee725122b91374b1782e93ae239aff04762 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 7 Mar 2024 19:11:12 -0800 Subject: [PATCH 386/459] Post-release CHANGELOG --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c8beeb..b40245a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## Git +### Added +### Fixed + + ## v0.10.0 - 2024-03-07 ### Added - Precompiled binaries for macOS ARM64 (darwin.aarch64) From 50db9a29c45f1f2a0db7ec60c8850c99e8e31d6e Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 7 Mar 2024 19:11:32 -0800 Subject: [PATCH 387/459] Check source details before git details --- test/check_release | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/test/check_release b/test/check_release index fd1dbca..4aef69a 100755 --- a/test/check_release +++ b/test/check_release @@ -12,6 +12,17 @@ then fail "There are uncommitted changes" fi +version=${current#v} +if ! grep "Version:" ShellCheck.cabal | grep -qFw "$version" +then + fail "The cabal file does not match tag version $version" +fi + +if ! grep -qF "## $current" CHANGELOG.md +then + fail "CHANGELOG.md does not contain '## $current'" +fi + current=$(git tag --points-at) if [[ -z "$current" ]] then @@ -34,17 +45,6 @@ then fail "You are not on master" fi -version=${current#v} -if ! grep "Version:" ShellCheck.cabal | grep -qFw "$version" -then - fail "The cabal file does not match tag version $version" -fi - -if ! grep -qF "## $current" CHANGELOG.md -then - fail "CHANGELOG.md does not contain '## $current'" -fi - if [[ $(git log -1 --pretty=%B) != "Stable version "* ]] then fail "Expected git log message to be 'Stable version ...'" From 9cb21c8557cb981e5f49da20af9335bb68f04dee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lawrence=20Vel=C3=A1zquez?= Date: Fri, 8 Mar 2024 18:24:08 -0500 Subject: [PATCH 388/459] Recommend `typeset` instead of `declare` in SC2324 Bash has both `typeset` and `declare`, but ksh has `typeset` only. Recommend the more portable alternative to users. --- src/ShellCheck/Analytics.hs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 1cc8bf8..f37ac1d 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -5017,7 +5017,8 @@ checkPlusEqualsNumber params t = state <- CF.getIncomingState cfga id guard $ isNumber state word guard . not $ fromMaybe False $ CF.variableMayBeDeclaredInteger state var - return $ warn id 2324 "var+=1 will append, not increment. Use (( var += 1 )), declare -i var, or quote number to silence." + -- Recommend "typeset" because ksh does not have "declare". + return $ warn id 2324 "var+=1 will append, not increment. Use (( var += 1 )), typeset -i var, or quote number to silence." _ -> return () where From 52dc66349b0882b0d6ac3d81a81f3eef0b155158 Mon Sep 17 00:00:00 2001 From: Joachim Ansorg Date: Tue, 12 Mar 2024 17:36:20 +0100 Subject: [PATCH 389/459] fix build of linux.aarch64 --- build/linux.aarch64/Dockerfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/build/linux.aarch64/Dockerfile b/build/linux.aarch64/Dockerfile index d5320e9..fadb6a4 100644 --- a/build/linux.aarch64/Dockerfile +++ b/build/linux.aarch64/Dockerfile @@ -12,11 +12,15 @@ RUN apt-get update && apt-get install -y llvm gcc-$TARGET # The rest are from 22.10 RUN sed -e 's/focal/kinetic/g' -i /etc/apt/sources.list +# Kinetic does not receive updates anymore, switch to last available +RUN sed -e 's/archive.ubuntu.com/old-releases.ubuntu.com/g' -i /etc/apt/sources.list +RUN sed -e 's/security.ubuntu.com/old-releases.ubuntu.com/g' -i /etc/apt/sources.list + RUN apt-get update && apt-get install -y ghc alex happy automake autoconf build-essential curl qemu-user-static # Build GHC WORKDIR /ghc -RUN curl -L "https://downloads.haskell.org/~ghc/9.2.5/ghc-9.2.5-src.tar.xz" | tar xJ --strip-components=1 +RUN curl -L "https://downloads.haskell.org/~ghc/9.2.8/ghc-9.2.8-src.tar.xz" | tar xJ --strip-components=1 RUN ./boot && ./configure --host x86_64-linux-gnu --build x86_64-linux-gnu --target "$TARGET" RUN cp mk/flavours/quick-cross.mk mk/build.mk && make -j "$(nproc)" RUN make install From c4123375e04931b3d0deb08728490687bc2fb3fd Mon Sep 17 00:00:00 2001 From: Joachim Ansorg Date: Tue, 12 Mar 2024 18:00:36 +0100 Subject: [PATCH 390/459] build smaller ShellCheck binary for Linux x86_64 --- build/linux.x86_64/Dockerfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/build/linux.x86_64/Dockerfile b/build/linux.x86_64/Dockerfile index 3112ac2..edafb36 100644 --- a/build/linux.x86_64/Dockerfile +++ b/build/linux.x86_64/Dockerfile @@ -1,4 +1,8 @@ -FROM alpine:latest +FROM alpine:3.16 +# alpine:3.16 (GHC 9.0.1): 5.8 megabytes +# alpine:3.17 (GHC 9.0.2): 15.0 megabytes +# alpine:3.18 (GHC 9.4.4): 29.0 megabytes +# alpine:3.19 (GHC 9.4.7): 29.0 megabytes ENV TARGETNAME linux.x86_64 From 0a7bb1822e9c80dae21aa6aaba6d8da3de5d9e94 Mon Sep 17 00:00:00 2001 From: Hugo Sousa <55895340+hugos99@users.noreply.github.com> Date: Thu, 4 Apr 2024 12:26:20 +0100 Subject: [PATCH 391/459] Update README.md to add macOS Arm64 pre-compiled binaries link --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 6542ac2..366c427 100644 --- a/README.md +++ b/README.md @@ -233,6 +233,7 @@ Alternatively, you can download pre-compiled binaries for the latest release her * [Linux, x86_64](https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.x86_64.tar.xz) (statically linked) * [Linux, armv6hf](https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.armv6hf.tar.xz), i.e. Raspberry Pi (statically linked) * [Linux, aarch64](https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.aarch64.tar.xz) aka ARM64 (statically linked) +* [macOS, aarch64](https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.darwin.aarch64.tar.xz) * [macOS, x86_64](https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.darwin.x86_64.tar.xz) * [Windows, x86](https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.zip) From 30b32af873c8b9e24731a5cb08bb20b7f148fa2d Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 4 Apr 2024 19:50:08 -0700 Subject: [PATCH 392/459] Add updating build images to release checks --- test/check_release | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/check_release b/test/check_release index 4aef69a..665b265 100755 --- a/test/check_release +++ b/test/check_release @@ -56,11 +56,14 @@ cat << EOF Manual Checklist $((i++)). Make sure none of the automated checks above failed +$((i++)). Run \`build/build_builder build/*/\` to update all builder images. +$((j++)). \`build/run_builder dist-newstyle/sdist/ShellCheck-*.tar.gz build/*/\` to verify that they work. +$((j++)). \`for f in \$(cat build/*/tag); do docker push "\$f"; done\` to upload them. +$((i++)). Run test/distrotest to ensure that most distros can build OOTB. $((i++)). Make sure GitHub Build currently passes: https://github.com/koalaman/shellcheck/actions $((i++)). Make sure SnapCraft build currently works: https://build.snapcraft.io/user/koalaman -$((i++)). Run test/distrotest to ensure that most distros can build OOTB. $((i++)). Format and read over the manual for bad formatting and outdated info. -$((i++)). Make sure the Hackage package builds. +$((i++)). Make sure the Hackage package builds locally. Release Steps From 5241878e5919d3581a8e0208c3d2345532dbb65f Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Fri, 5 Apr 2024 17:15:04 -0700 Subject: [PATCH 393/459] Update Windows build image with new cURL URL --- build/windows.x86_64/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/windows.x86_64/Dockerfile b/build/windows.x86_64/Dockerfile index 1e5c5d9..2ae78ac 100644 --- a/build/windows.x86_64/Dockerfile +++ b/build/windows.x86_64/Dockerfile @@ -12,7 +12,7 @@ WORKDIR /haskell RUN curl -L "https://downloads.haskell.org/~ghc/8.10.4/ghc-8.10.4-x86_64-unknown-mingw32.tar.xz" | tar xJ --strip-components=1 WORKDIR /haskell/bin RUN curl -L "https://downloads.haskell.org/~cabal/cabal-install-3.2.0.0/cabal-install-3.2.0.0-x86_64-unknown-mingw32.zip" | busybox unzip - -RUN curl -L "https://curl.se/windows/dl-7.84.0/curl-7.84.0-win64-mingw.zip" | busybox unzip - && mv curl-7.84.0-win64-mingw/bin/* . +RUN curl -L "https://curl.se/windows/dl-8.7.1_7/curl-8.7.1_7-win64-mingw.zip" | busybox unzip - && mv curl-*-win64-mingw/bin/* . ENV WINEPATH /haskell/bin # It's unknown whether Cabal on Windows suffers from the same issue From 04a86245a10fe0a9e48755237f315999986f54c0 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 8 Apr 2024 20:24:28 -0700 Subject: [PATCH 394/459] Remove trailing space in output (fixes #2961) --- src/ShellCheck/Formatter/TTY.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Formatter/TTY.hs b/src/ShellCheck/Formatter/TTY.hs index e503639..117da6e 100644 --- a/src/ShellCheck/Formatter/TTY.hs +++ b/src/ShellCheck/Formatter/TTY.hs @@ -169,7 +169,7 @@ showFixedString color comments lineNum fileLines = -- and/or other unrelated lines. let (excerptFix, excerpt) = sliceFile mergedFix fileLines -- in the spirit of error prone - putStrLn $ color "message" "Did you mean: " + putStrLn $ color "message" "Did you mean:" putStrLn $ unlines $ applyFix excerptFix excerpt cuteIndent :: PositionedComment -> String From 2c5155e43d030e1325c3c2765d8f492024b02fd9 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 14 Apr 2024 18:47:19 -0700 Subject: [PATCH 395/459] Warn about capturing the output of redirected commands. --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 42 +++++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b40245a..25fc688 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ## Git ### Added +- SC2327/SC2328: Warn about capturing the output of redirected commands. ### Fixed diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index f37ac1d..8b74ac6 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -204,6 +204,7 @@ nodeChecks = [ ,checkUnnecessaryArithmeticExpansionIndex ,checkUnnecessaryParens ,checkPlusEqualsNumber + ,checkExpansionWithRedirection ] optionalChecks = map fst optionalTreeChecks @@ -5040,5 +5041,46 @@ checkPlusEqualsNumber params t = isUnquotedNumber || isNumericalVariableName || isNumericalVariableExpansion + +prop_checkExpansionWithRedirection1 = verify checkExpansionWithRedirection "var=$(foo > bar)" +prop_checkExpansionWithRedirection2 = verify checkExpansionWithRedirection "var=`foo 1> bar`" +prop_checkExpansionWithRedirection3 = verify checkExpansionWithRedirection "var=${ foo >> bar; }" +prop_checkExpansionWithRedirection4 = verify checkExpansionWithRedirection "var=$(foo | bar > baz)" +prop_checkExpansionWithRedirection5 = verifyNot checkExpansionWithRedirection "stderr=$(foo 2>&1 > /dev/null)" +prop_checkExpansionWithRedirection6 = verifyNot checkExpansionWithRedirection "var=$(foo; bar > baz)" +prop_checkExpansionWithRedirection7 = verifyNot checkExpansionWithRedirection "var=$(foo > bar; baz)" +prop_checkExpansionWithRedirection8 = verifyNot checkExpansionWithRedirection "var=$(cat <&3)" +checkExpansionWithRedirection params t = + case t of + T_DollarExpansion id [cmd] -> check id cmd + T_Backticked id [cmd] -> check id cmd + T_DollarBraceCommandExpansion id [cmd] -> check id cmd + _ -> return () + where + check id pipe = + case pipe of + (T_Pipeline _ _ t@(_:_)) -> checkCmd id (last t) + _ -> return () + + checkCmd captureId (T_Redirecting _ redirs _) = walk captureId redirs + + walk captureId [] = return () + walk captureId (t:rest) = + case t of + T_FdRedirect _ _ (T_IoDuplicate _ _ "1") -> return () + T_FdRedirect id "1" (T_IoDuplicate _ _ _) -> return () + T_FdRedirect id "" (T_IoDuplicate _ op _) | op `elem` [T_GREATAND (Id 0), T_Greater (Id 0)] -> emit id captureId True + T_FdRedirect id str (T_IoFile _ op file) | str `elem` ["", "1"] && op `elem` [ T_DGREAT (Id 0), T_Greater (Id 0) ] -> + if getLiteralString file == Just "/dev/null" + then emit id captureId False + else emit id captureId True + _ -> walk captureId rest + + emit redirectId captureId suggestTee = do + warn captureId 2327 "This command substitution will be empty because the command's output gets redirected away." + err redirectId 2328 $ "This redirection takes output away from the command substitution" ++ if suggestTee then " (use tee to duplicate)." else "." + + + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) From 69fe4e1306572fa639d92e80afe2639d6c12247f Mon Sep 17 00:00:00 2001 From: Syuugo Date: Thu, 25 Apr 2024 10:35:43 +0900 Subject: [PATCH 396/459] Upgrade build workflow dependencies --- .github/workflows/build.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c49b25a..378a0cf 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,7 +15,7 @@ jobs: sudo apt-get install cabal-install - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 @@ -37,7 +37,7 @@ jobs: mv dist-newstyle/sdist/*.tar.gz source/source.tar.gz - name: Upload artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: source path: source/ @@ -51,10 +51,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Download artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 - name: Build source run: | @@ -63,7 +63,7 @@ jobs: ( cd bin && ../build/run_builder ../source/source.tar.gz ../build/${{matrix.build}} ) - name: Upload artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: bin path: bin/ @@ -74,10 +74,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Download artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 - name: Work around GitHub permissions bug run: chmod +x bin/*/shellcheck* @@ -92,7 +92,7 @@ jobs: rm -rf */ README* LICENSE* - name: Upload artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: deploy path: deploy/ @@ -109,10 +109,10 @@ jobs: sudo apt-get install hub - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Download artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 - name: Upload to GitHub env: From 796c6bd8482e4666c4f73d2874c3949c2bed801e Mon Sep 17 00:00:00 2001 From: Jan Dubois Date: Wed, 24 Apr 2024 19:05:29 -0700 Subject: [PATCH 397/459] Add new bats variables stderr and stderr_lines These are being set by `run --separate-stderr` and have been introduced in https://github.com/bats-core/bats-core/releases/tag/v1.5.0 --- src/ShellCheck/AnalyzerLib.hs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index d265ace..a2d61d2 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -535,7 +535,9 @@ getModifiedVariables t = T_BatsTest {} -> [ (t, t, "lines", DataArray SourceExternal), (t, t, "status", DataString SourceInteger), - (t, t, "output", DataString SourceExternal) + (t, t, "output", DataString SourceExternal), + (t, t, "stderr", DataString SourceExternal), + (t, t, "stderr_lines", DataArray SourceExternal) ] -- Count [[ -v foo ]] as an "assignment". From 4f81dbe839091a06a5cfaea695cf1c451ff07565 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 4 May 2024 14:21:12 -0700 Subject: [PATCH 398/459] Add warning about uninvoked functions, reduce repeated triggering of SC2317 (fixes #2966) --- CHANGELOG.md | 2 ++ src/ShellCheck/Analytics.hs | 23 ++++++++++++++++++++--- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 25fc688..5277893 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,9 @@ ## Git ### Added - SC2327/SC2328: Warn about capturing the output of redirected commands. +- SC2329: Warn when (non-escaping) functions are never invoked. ### Fixed +- SC2317 about unreachable commands is now less spammy for nested ones. ## v0.10.0 - 2024-03-07 diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 8b74ac6..685fbf4 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -4896,16 +4896,33 @@ checkBatsTestDoesNotUseNegation params t = prop_checkCommandIsUnreachable1 = verify checkCommandIsUnreachable "foo; bar; exit; baz" prop_checkCommandIsUnreachable2 = verify checkCommandIsUnreachable "die() { exit; }; foo; bar; die; baz" prop_checkCommandIsUnreachable3 = verifyNot checkCommandIsUnreachable "foo; bar || exit; baz" +prop_checkCommandIsUnreachable4 = verifyNot checkCommandIsUnreachable "f() { foo; }; # Maybe sourced" +prop_checkCommandIsUnreachable5 = verify checkCommandIsUnreachable "f() { foo; }; exit # Not sourced" checkCommandIsUnreachable params t = case t of T_Pipeline {} -> sequence_ $ do cfga <- cfgAnalysis params - state <- CF.getIncomingState cfga id + state <- CF.getIncomingState cfga (getId t) guard . not $ CF.stateIsReachable state guard . not $ isSourced params t - return $ info id 2317 "Command appears to be unreachable. Check usage (or ignore if invoked indirectly)." + guard . not $ any (\t -> isUnreachable t || isUnreachableFunction t) $ NE.drop 1 $ getPath (parentMap params) t + return $ info (getId t) 2317 "Command appears to be unreachable. Check usage (or ignore if invoked indirectly)." + T_Function id _ _ _ _ -> + when (isUnreachableFunction t + && (not . any isUnreachableFunction . NE.drop 1 $ getPath (parentMap params) t) + && (not $ isSourced params t)) $ + info id 2329 "This function is never invoked. Check usage (or ignored if invoked indirectly)." _ -> return () - where id = getId t + where + isUnreachableFunction :: Token -> Bool + isUnreachableFunction f = + case f of + T_Function id _ _ _ t -> isUnreachable t + _ -> False + isUnreachable t = fromMaybe False $ do + cfga <- cfgAnalysis params + state <- CF.getIncomingState cfga (getId t) + return . not $ CF.stateIsReachable state prop_checkOverwrittenExitCode1 = verify checkOverwrittenExitCode "x; [ $? -eq 1 ] || [ $? -eq 2 ]" From 76ff702e9385215a888ed21bf4330f614ab6c355 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 4 May 2024 15:12:13 -0700 Subject: [PATCH 399/459] Supress SC2015 about `A && B || C` when B is a test. --- CHANGELOG.md | 2 ++ src/ShellCheck/Analytics.hs | 16 +++++----------- src/ShellCheck/AnalyzerLib.hs | 8 ++++++++ 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5277893..fe38b8a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ### Added - SC2327/SC2328: Warn about capturing the output of redirected commands. - SC2329: Warn when (non-escaping) functions are never invoked. +### Changed +- SC2015 about `A && B || C` no longer triggers when B is a test command. ### Fixed - SC2317 about unreachable commands is now less spammy for nested ones. diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 685fbf4..175dea6 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -877,8 +877,9 @@ prop_checkShorthandIf5 = verifyNot checkShorthandIf "foo && rm || printf b" prop_checkShorthandIf6 = verifyNot checkShorthandIf "if foo && bar || baz; then true; fi" prop_checkShorthandIf7 = verifyNot checkShorthandIf "while foo && bar || baz; do true; done" prop_checkShorthandIf8 = verify checkShorthandIf "if true; then foo && bar || baz; fi" -checkShorthandIf params x@(T_OrIf _ (T_AndIf id _ _) (T_Pipeline _ _ t)) - | not (isOk t || inCondition) = +prop_checkShorthandIf9 = verifyNot checkShorthandIf "foo && [ -x /file ] || bar" +checkShorthandIf params x@(T_OrIf _ (T_AndIf id _ b) (T_Pipeline _ _ t)) + | not (isOk t || inCondition) && not (isTestCommand b) = info id 2015 "Note that A && B || C is not if-then-else. C may run when A is true." where isOk [t] = isAssignment t || fromMaybe False (do @@ -4197,7 +4198,7 @@ checkBadTestAndOr params t = in mapM_ checkTest commandWithSeps checkTest (before, cmd, after) = - when (isTest cmd) $ do + when (isTestCommand cmd) $ do checkPipe before checkPipe after @@ -4213,17 +4214,10 @@ checkBadTestAndOr params t = T_AndIf _ _ rhs -> checkAnds id rhs T_OrIf _ _ rhs -> checkAnds id rhs T_Pipeline _ _ list | not (null list) -> checkAnds id (last list) - cmd -> when (isTest cmd) $ + cmd -> when (isTestCommand cmd) $ errWithFix id 2265 "Use && for logical AND. Single & will background and return true." $ (fixWith [replaceEnd id params 0 "&"]) - isTest t = - case t of - T_Condition {} -> True - T_SimpleCommand {} -> t `isCommand` "test" - T_Redirecting _ _ t -> isTest t - T_Annotation _ _ t -> isTest t - _ -> False prop_checkComparisonWithLeadingX1 = verify checkComparisonWithLeadingX "[ x$foo = xlol ]" prop_checkComparisonWithLeadingX2 = verify checkComparisonWithLeadingX "test x$foo = xlol" diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index d265ace..c6e4e14 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -929,6 +929,14 @@ modifiesVariable params token name = Assignment (_, _, n, source) -> isTrueAssignmentSource source && n == name _ -> False +isTestCommand t = + case t of + T_Condition {} -> True + T_SimpleCommand {} -> t `isCommand` "test" + T_Redirecting _ _ t -> isTestCommand t + T_Annotation _ _ t -> isTestCommand t + T_Pipeline _ _ [t] -> isTestCommand t + _ -> False return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) From d705716dc45d58eef51231292758e2af9e3da30b Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 4 May 2024 15:22:09 -0700 Subject: [PATCH 400/459] Account for annotations in SC2215. Fixes #2975. --- src/ShellCheck/Analytics.hs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 175dea6..dcb4ce8 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -4000,6 +4000,7 @@ prop_checkUselessBang6 = verify checkUselessBang "set -e; { ! true; }" prop_checkUselessBang7 = verifyNot checkUselessBang "set -e; x() { ! [ x ]; }" prop_checkUselessBang8 = verifyNot checkUselessBang "set -e; if { ! true; }; then true; fi" prop_checkUselessBang9 = verifyNot checkUselessBang "set -e; while ! true; do true; done" +prop_checkUselessBang10 = verify checkUselessBang "set -e\nshellcheck disable=SC0000\n! true\nrest" checkUselessBang params t = when (hasSetE params) $ mapM_ check (getNonReturningCommands t) where check t = @@ -4008,6 +4009,7 @@ checkUselessBang params t = when (hasSetE params) $ mapM_ check (getNonReturning addComment $ makeCommentWithFix InfoC id 2251 "This ! is not on a condition and skips errexit. Use `&& exit 1` instead, or make sure $? is checked." (fixWith [replaceStart id params 1 "", replaceEnd (getId cmd) params 0 " && exit 1"]) + T_Annotation _ _ t -> check t _ -> return () -- Get all the subcommands that aren't likely to be the return value From a7a906e2cbca41611f232208ae062fe7ddc719f7 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 4 May 2024 16:28:56 -0700 Subject: [PATCH 401/459] Allow SC2154 to trigger in arrays (fixes #2970) --- src/ShellCheck/Analytics.hs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index dcb4ce8..dc84a78 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2447,6 +2447,7 @@ prop_checkUnassignedReferences_minusZDefault = verifyNotTree checkUnassignedRefe prop_checkUnassignedReferences50 = verifyNotTree checkUnassignedReferences "echo ${foo:+bar}" prop_checkUnassignedReferences51 = verifyNotTree checkUnassignedReferences "echo ${foo:+$foo}" prop_checkUnassignedReferences52 = verifyNotTree checkUnassignedReferences "wait -p pid; echo $pid" +prop_checkUnassignedReferences53 = verify checkUnassignedReferences "x=($foo)" checkUnassignedReferences = checkUnassignedReferences' False checkUnassignedReferences' includeGlobals params t = warnings @@ -2502,14 +2503,12 @@ checkUnassignedReferences' includeGlobals params t = warnings warnings = execWriter . sequence $ mapMaybe warningFor unassigned - -- Due to parsing, foo=( [bar]=baz ) parses 'bar' as a reference even for assoc arrays. - -- Similarly, ${foo[bar baz]} may not be referencing bar/baz. Just skip these. + -- ${foo[bar baz]} may not be referencing bar/baz. Just skip these. -- We can also have ${foo:+$foo} should be treated like [[ -n $foo ]] && echo $foo isException var t = any shouldExclude $ getPath (parentMap params) t where shouldExclude t = case t of - T_Array {} -> True (T_DollarBraced _ _ l) -> let str = concat $ oversimplify l ref = getBracedReference str From a13cb85f49c074ce6ab4644beaf8665a3ff6f395 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 4 May 2024 16:34:21 -0700 Subject: [PATCH 402/459] Fixed broken test due to bad build cache --- src/ShellCheck/Analytics.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index dc84a78..f5e57e2 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2447,7 +2447,7 @@ prop_checkUnassignedReferences_minusZDefault = verifyNotTree checkUnassignedRefe prop_checkUnassignedReferences50 = verifyNotTree checkUnassignedReferences "echo ${foo:+bar}" prop_checkUnassignedReferences51 = verifyNotTree checkUnassignedReferences "echo ${foo:+$foo}" prop_checkUnassignedReferences52 = verifyNotTree checkUnassignedReferences "wait -p pid; echo $pid" -prop_checkUnassignedReferences53 = verify checkUnassignedReferences "x=($foo)" +prop_checkUnassignedReferences53 = verifyTree checkUnassignedReferences "x=($foo)" checkUnassignedReferences = checkUnassignedReferences' False checkUnassignedReferences' includeGlobals params t = warnings From ac8fb00504ed6da83fe5c5f83e72e4663ff6b439 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 4 May 2024 16:45:52 -0700 Subject: [PATCH 403/459] Account for BusyBox support of [[ ]] (fixes #2967) --- CHANGELOG.md | 2 ++ src/ShellCheck/Analytics.hs | 11 ++++++++--- src/ShellCheck/AnalyzerLib.hs | 10 ---------- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fe38b8a..8ae176d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,10 +2,12 @@ ### Added - SC2327/SC2328: Warn about capturing the output of redirected commands. - SC2329: Warn when (non-escaping) functions are never invoked. +- SC2330: Warn about unsupported glob matches with [[ .. ]] in BusyBox. ### Changed - SC2015 about `A && B || C` no longer triggers when B is a test command. ### Fixed - SC2317 about unreachable commands is now less spammy for nested ones. +- SC2292, optional suggestion for [[ ]], now triggers for Busybox. ## v0.10.0 - 2024-03-07 diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index f5e57e2..a89f940 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1525,6 +1525,7 @@ prop_checkComparisonAgainstGlob3 = verify checkComparisonAgainstGlob "[ $cow = * prop_checkComparisonAgainstGlob4 = verifyNot checkComparisonAgainstGlob "[ $cow = foo ]" prop_checkComparisonAgainstGlob5 = verify checkComparisonAgainstGlob "[[ $cow != $bar ]]" prop_checkComparisonAgainstGlob6 = verify checkComparisonAgainstGlob "[ $f != /* ]" +prop_checkComparisonAgainstGlob7 = verify checkComparisonAgainstGlob "#!/bin/busybox sh\n[[ $f == *foo* ]]" checkComparisonAgainstGlob _ (TC_Binary _ DoubleBracket op _ (T_NormalWord id [T_DollarBraced _ _ _])) | op `elem` ["=", "==", "!="] = warn id 2053 $ "Quote the right-hand side of " ++ op ++ " in [[ ]] to prevent glob matching." @@ -1532,10 +1533,14 @@ checkComparisonAgainstGlob params (TC_Binary _ SingleBracket op _ word) | op `elem` ["=", "==", "!="] && isGlob word = err (getId word) 2081 msg where - msg = if isBashLike params + msg = if (shellType params) `elem` [Bash, Ksh] -- Busybox does not support glob matching then "[ .. ] can't match globs. Use [[ .. ]] or case statement." else "[ .. ] can't match globs. Use a case statement." +checkComparisonAgainstGlob params (TC_Binary _ DoubleBracket op _ word) + | shellType params == BusyboxSh && op `elem` ["=", "==", "!="] && isGlob word = + err (getId word) 2330 "BusyBox [[ .. ]] does not support glob matching. Use a case statement." + checkComparisonAgainstGlob _ _ = return () prop_checkCaseAgainstGlob1 = verify checkCaseAgainstGlob "case foo in lol$n) foo;; esac" @@ -4534,13 +4539,13 @@ prop_checkRequireDoubleBracket2 = verifyTree checkRequireDoubleBracket "[ foo -o prop_checkRequireDoubleBracket3 = verifyNotTree checkRequireDoubleBracket "#!/bin/sh\n[ -x foo ]" prop_checkRequireDoubleBracket4 = verifyNotTree checkRequireDoubleBracket "[[ -x foo ]]" checkRequireDoubleBracket params = - if isBashLike params + if (shellType params) `elem` [Bash, Ksh, BusyboxSh] then nodeChecksToTreeCheck [check] params else const [] where check _ t = case t of T_Condition id SingleBracket _ -> - styleWithFix id 2292 "Prefer [[ ]] over [ ] for tests in Bash/Ksh." (fixFor t) + styleWithFix id 2292 "Prefer [[ ]] over [ ] for tests in Bash/Ksh/Busybox." (fixFor t) _ -> return () fixFor t = fixWith $ diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index c6e4e14..cae73b1 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -902,16 +902,6 @@ supportsArrays Bash = True supportsArrays Ksh = True supportsArrays _ = False --- Returns true if the shell is Bash or Ksh (sorry for the name, Ksh) -isBashLike :: Parameters -> Bool -isBashLike params = - case shellType params of - Bash -> True - Ksh -> True - Dash -> False - BusyboxSh -> False - Sh -> False - isTrueAssignmentSource c = case c of DataString SourceChecked -> False From 78d1ee0222a114597abba1ca8f0784b673bf7d97 Mon Sep 17 00:00:00 2001 From: Bryan Honof Date: Fri, 24 May 2024 17:15:09 +0200 Subject: [PATCH 404/459] Add Flox to list of installation methods --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 366c427..754f897 100644 --- a/README.md +++ b/README.md @@ -228,6 +228,11 @@ Using the [nix package manager](https://nixos.org/nix): nix-env -iA nixpkgs.shellcheck ``` +Using the [Flox package manager](https://flox.dev/) +```sh +flox install shellcheck +``` + Alternatively, you can download pre-compiled binaries for the latest release here: * [Linux, x86_64](https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.x86_64.tar.xz) (statically linked) From 15de97e33f462434272f13e1ec3aa9cd5387441b Mon Sep 17 00:00:00 2001 From: Meng Zhuo Date: Thu, 30 May 2024 19:20:21 +0800 Subject: [PATCH 405/459] Add linux.riscv64 precompiled support --- .github/workflows/build.yml | 2 +- .multi_arch_docker | 1 + CHANGELOG.md | 1 + build/linux.riscv64/Dockerfile | 47 ++++++++++++++++++++++++++++++++++ build/linux.riscv64/build | 15 +++++++++++ build/linux.riscv64/tag | 1 + 6 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 build/linux.riscv64/Dockerfile create mode 100755 build/linux.riscv64/build create mode 100644 build/linux.riscv64/tag diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c49b25a..e30b2ac 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -47,7 +47,7 @@ jobs: needs: package_source strategy: matrix: - build: [linux.x86_64, linux.aarch64, linux.armv6hf, darwin.x86_64, darwin.aarch64, windows.x86_64] + build: [linux.x86_64, linux.aarch64, linux.armv6hf, linux.riscv64, darwin.x86_64, darwin.aarch64, windows.x86_64] runs-on: ubuntu-latest steps: - name: Checkout repository diff --git a/.multi_arch_docker b/.multi_arch_docker index 1c5d32b..81048a2 100755 --- a/.multi_arch_docker +++ b/.multi_arch_docker @@ -80,6 +80,7 @@ function multi_arch_docker::main() { export DOCKER_PLATFORMS='linux/amd64' DOCKER_PLATFORMS+=' linux/arm64' DOCKER_PLATFORMS+=' linux/arm/v6' + DOCKER_PLATFORMS+=' linux/riscv64' multi_arch_docker::install_docker_buildx multi_arch_docker::login_to_docker_hub diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ae176d..43db00c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ - SC2327/SC2328: Warn about capturing the output of redirected commands. - SC2329: Warn when (non-escaping) functions are never invoked. - SC2330: Warn about unsupported glob matches with [[ .. ]] in BusyBox. +- Precompiled binaries for Linux riscv64 (linux.riscv64) ### Changed - SC2015 about `A && B || C` no longer triggers when B is a test command. ### Fixed diff --git a/build/linux.riscv64/Dockerfile b/build/linux.riscv64/Dockerfile new file mode 100644 index 0000000..648ef2d --- /dev/null +++ b/build/linux.riscv64/Dockerfile @@ -0,0 +1,47 @@ +FROM ubuntu:22.04 + +ENV TARGETNAME linux.riscv64 +ENV TARGET riscv64-linux-gnu + +USER root +ENV DEBIAN_FRONTEND noninteractive + +# Init base +RUN apt update -y + +# Install qemu +RUN apt install -y --no-install-recommends build-essential ninja-build python3 pkg-config libglib2.0-dev libpixman-1-dev curl ca-certificates python3-virtualenv +WORKDIR /qemu +RUN curl -Lv "https://download.qemu.org/qemu-9.0.0.tar.xz" | tar xJ --strip-components=1 +RUN ./configure --target-list=riscv64-linux-user --static --disable-system --disable-pie +RUN cd build && ninja qemu-riscv64 +ENV QEMU_EXECVE 1 + + +# Set up a riscv64 userspace +RUN apt install -y --no-install-recommends debootstrap +RUN debootstrap --arch=riscv64 --foreign jammy /rvfs http://ports.ubuntu.com/ubuntu-ports +RUN cp /qemu/build/qemu-riscv64 /rvfs/usr/bin/qemu-riscv64-static + +RUN printf > /bin/rv '%s\n' '#!/bin/sh' 'chroot /rvfs /usr/bin/qemu-riscv64-static /usr/bin/env "$@"' +RUN chmod +x /bin/rv +RUN [ ! -e /rvfs/debootstrap ] || rv '/debootstrap/debootstrap' --second-stage + +# Install deps in the chroot +RUN printf > /rvfs/etc/apt/sources.list '%s\n' 'deb http://ports.ubuntu.com/ubuntu-ports jammy main universe' +RUN rv apt update -y +RUN rv apt install -y --no-install-recommends ghc cabal-install + +# Finally we can build the current dependencies. This takes hours. +# jobs must be 1, GHS riscv will use about 40G memory +RUN rv cabal update +RUN IFS=";" && rv cabal install --dependencies-only --jobs=1 ShellCheck +RUN IFS=';' && rv cabal install --lib --jobs=1 fgl + +# Clean up +RUN rm -rf /qemu + +# Copy the build script +WORKDIR /rvfs/scratch +COPY build /rvfs/usr/bin/build +ENTRYPOINT ["/bin/rv", "/usr/bin/build"] diff --git a/build/linux.riscv64/build b/build/linux.riscv64/build new file mode 100755 index 0000000..19cb143 --- /dev/null +++ b/build/linux.riscv64/build @@ -0,0 +1,15 @@ +#!/bin/sh +set -xe +{ + tar xzv --strip-components=1 + chmod +x striptests && ./striptests + mkdir "$TARGETNAME" + cabal update + ( IFS=';'; cabal build --enable-executable-static ) + find . -name shellcheck -type f -exec mv {} "$TARGETNAME/" \; + ls -l "$TARGETNAME" + "$TARGET-strip" -s "$TARGETNAME/shellcheck" + ls -l "$TARGETNAME" + qemu-riscv64-static "$TARGETNAME/shellcheck" --version +} >&2 +tar czv "$TARGETNAME" diff --git a/build/linux.riscv64/tag b/build/linux.riscv64/tag new file mode 100644 index 0000000..901eaaa --- /dev/null +++ b/build/linux.riscv64/tag @@ -0,0 +1 @@ +koalaman/scbuilder-linux-riscv64 From 23e76de4f2a640ecd7c8c1da23495d985ef095e6 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 11 Jun 2024 06:00:22 +0000 Subject: [PATCH 406/459] Allow riscv64 image to run without binfmt_misc --- build/linux.riscv64/Dockerfile | 44 ++++++----- build/linux.riscv64/build | 10 ++- build/linux.riscv64/cabal.project.freeze | 93 ++++++++++++++++++++++++ 3 files changed, 126 insertions(+), 21 deletions(-) create mode 100644 build/linux.riscv64/cabal.project.freeze diff --git a/build/linux.riscv64/Dockerfile b/build/linux.riscv64/Dockerfile index 648ef2d..0ac95e5 100644 --- a/build/linux.riscv64/Dockerfile +++ b/build/linux.riscv64/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:22.04 +FROM ubuntu:24.04 ENV TARGETNAME linux.riscv64 ENV TARGET riscv64-linux-gnu @@ -7,39 +7,47 @@ USER root ENV DEBIAN_FRONTEND noninteractive # Init base -RUN apt update -y +RUN apt-get update -y # Install qemu -RUN apt install -y --no-install-recommends build-essential ninja-build python3 pkg-config libglib2.0-dev libpixman-1-dev curl ca-certificates python3-virtualenv +RUN apt-get install -y --no-install-recommends build-essential ninja-build python3 pkg-config libglib2.0-dev libpixman-1-dev curl ca-certificates python3-virtualenv git python3-setuptools WORKDIR /qemu -RUN curl -Lv "https://download.qemu.org/qemu-9.0.0.tar.xz" | tar xJ --strip-components=1 -RUN ./configure --target-list=riscv64-linux-user --static --disable-system --disable-pie +RUN git clone --depth 1 https://github.com/koalaman/qemu . +#RUN git clone https://github.com/balena-io/qemu . +# Release 7.0.0 +#RUN git checkout 639d1d8903f65d74eb04c49e0df7a4b2f014cd86 +RUN ./configure --target-list=riscv64-linux-user --static --disable-system --disable-pie --disable-werror RUN cd build && ninja qemu-riscv64 ENV QEMU_EXECVE 1 # Set up a riscv64 userspace -RUN apt install -y --no-install-recommends debootstrap -RUN debootstrap --arch=riscv64 --foreign jammy /rvfs http://ports.ubuntu.com/ubuntu-ports +RUN apt-get install -y --no-install-recommends debootstrap +RUN debootstrap --arch=riscv64 --foreign noble /rvfs http://ports.ubuntu.com/ubuntu-ports RUN cp /qemu/build/qemu-riscv64 /rvfs/usr/bin/qemu-riscv64-static -RUN printf > /bin/rv '%s\n' '#!/bin/sh' 'chroot /rvfs /usr/bin/qemu-riscv64-static /usr/bin/env "$@"' +# Command to run riscv binaries in the chroot. The Haskell runtime allocates 1TB +# vspace up front and QEmu has a RAM cost per vspace, so use ulimit to allocate +# less and reduce RAM usage. +RUN printf > /bin/rv '%s\n' '#!/bin/sh' 'ulimit -v $((10*1024*1024)); chroot /rvfs /usr/bin/qemu-riscv64-static /usr/bin/env "$@"' RUN chmod +x /bin/rv RUN [ ! -e /rvfs/debootstrap ] || rv '/debootstrap/debootstrap' --second-stage # Install deps in the chroot -RUN printf > /rvfs/etc/apt/sources.list '%s\n' 'deb http://ports.ubuntu.com/ubuntu-ports jammy main universe' -RUN rv apt update -y -RUN rv apt install -y --no-install-recommends ghc cabal-install +RUN printf > /rvfs/etc/apt/sources.list '%s\n' 'deb http://ports.ubuntu.com/ubuntu-ports noble main universe' +RUN rv apt-get update -y +RUN rv apt-get install -y --no-install-recommends ghc cabal-install -# Finally we can build the current dependencies. This takes hours. -# jobs must be 1, GHS riscv will use about 40G memory RUN rv cabal update -RUN IFS=";" && rv cabal install --dependencies-only --jobs=1 ShellCheck -RUN IFS=';' && rv cabal install --lib --jobs=1 fgl - -# Clean up -RUN rm -rf /qemu +# Generated with: cabal freeze -c 'hashable -arch-native'. We put it in /etc so cabal won't find it. +COPY cabal.project.freeze /rvfs/etc +# Awful hack to install everything from the freeze file +# This basically turns 'any.tagged ==0.8.8' into tagged-0.8.8 to install by version, +# and adds a -c before 'hashable -arch-native +integer-gmp' to make it a flag constraint. +RUN < /rvfs/etc/cabal.project.freeze sed 's/constraints:/&\n /' | grep -vw rts | sed -n -e 's/^ *\([^,]*\).*/\1/p' | sed -e 's/any\.\([^ ]*\) ==\(.*\)/\1-\2/; te; s/.*/-c\n&/; :e' > /tmp/preinstall-flags +# Finally we can build the current dependencies. This takes hours. +# There's apparently a random segfault during assembly, so retry a few times. +RUN set -x; IFS=${IFS# }; f() { rv cabal install --keep-going $(cat /tmp/preinstall-flags); }; for i in $(seq 5); do f; ret=$?; [ $ret = 0 ] && break; done; exit $ret # Copy the build script WORKDIR /rvfs/scratch diff --git a/build/linux.riscv64/build b/build/linux.riscv64/build index 19cb143..63c487f 100755 --- a/build/linux.riscv64/build +++ b/build/linux.riscv64/build @@ -1,15 +1,19 @@ #!/bin/sh set -xe +IFS=';' { tar xzv --strip-components=1 chmod +x striptests && ./striptests + # Use a freeze file to ensure we use the same dependencies we cached during + # the docker image build. We don't want to spend time compiling anything new. + cp /etc/cabal.project.freeze . mkdir "$TARGETNAME" - cabal update - ( IFS=';'; cabal build --enable-executable-static ) + # Retry in case of random segfault + cabal build --enable-executable-static || cabal build --enable-executable-static find . -name shellcheck -type f -exec mv {} "$TARGETNAME/" \; ls -l "$TARGETNAME" "$TARGET-strip" -s "$TARGETNAME/shellcheck" ls -l "$TARGETNAME" - qemu-riscv64-static "$TARGETNAME/shellcheck" --version + "$TARGETNAME/shellcheck" --version } >&2 tar czv "$TARGETNAME" diff --git a/build/linux.riscv64/cabal.project.freeze b/build/linux.riscv64/cabal.project.freeze new file mode 100644 index 0000000..cbb42e1 --- /dev/null +++ b/build/linux.riscv64/cabal.project.freeze @@ -0,0 +1,93 @@ +active-repositories: hackage.haskell.org:merge +constraints: any.Diff ==0.5, + any.OneTuple ==0.4.2, + any.QuickCheck ==2.14.3, + QuickCheck -old-random +templatehaskell, + any.StateVar ==1.2.2, + any.aeson ==2.2.3.0, + aeson +ordered-keymap, + any.array ==0.5.4.0, + any.assoc ==1.1.1, + assoc -tagged, + any.base ==4.17.2.0, + any.base-orphans ==0.9.2, + any.bifunctors ==5.6.2, + bifunctors +tagged, + any.binary ==0.8.9.1, + any.bytestring ==0.11.5.2, + any.character-ps ==0.1, + any.comonad ==5.0.8, + comonad +containers +distributive +indexed-traversable, + any.containers ==0.6.7, + any.contravariant ==1.5.5, + contravariant +semigroups +statevar +tagged, + any.data-fix ==0.3.3, + any.deepseq ==1.4.8.0, + any.directory ==1.3.7.1, + any.distributive ==0.6.2.1, + distributive +semigroups +tagged, + any.dlist ==1.0, + dlist -werror, + any.exceptions ==0.10.5, + any.fgl ==5.8.2.0, + fgl +containers042, + any.filepath ==1.4.2.2, + any.foldable1-classes-compat ==0.1, + foldable1-classes-compat +tagged, + any.generically ==0.1.1, + any.ghc-bignum ==1.3, + any.ghc-boot-th ==9.4.7, + any.ghc-prim ==0.9.1, + any.hashable ==1.4.6.0, + hashable -arch-native +integer-gmp -random-initial-seed, + any.indexed-traversable ==0.1.4, + any.indexed-traversable-instances ==0.1.2, + any.integer-conversion ==0.1.1, + any.integer-logarithms ==1.0.3.1, + integer-logarithms -check-bounds +integer-gmp, + any.mtl ==2.2.2, + any.network-uri ==2.6.4.2, + any.os-string ==2.0.3, + any.parsec ==3.1.16.1, + any.pretty ==1.1.3.6, + any.primitive ==0.9.0.0, + any.process ==1.6.17.0, + any.random ==1.2.1.2, + any.regex-base ==0.94.0.2, + any.regex-tdfa ==1.3.2.2, + regex-tdfa +doctest -force-o2, + any.rts ==1.0.2, + any.scientific ==0.3.8.0, + scientific -integer-simple, + any.semialign ==1.3.1, + semialign +semigroupoids, + any.semigroupoids ==6.0.1, + semigroupoids +comonad +containers +contravariant +distributive +tagged +unordered-containers, + any.splitmix ==0.1.0.5, + splitmix -optimised-mixer, + any.stm ==2.5.1.0, + any.strict ==0.5, + any.tagged ==0.8.8, + tagged +deepseq +transformers, + any.template-haskell ==2.19.0.0, + any.text ==2.0.2, + any.text-iso8601 ==0.1.1, + any.text-short ==0.1.6, + text-short -asserts, + any.th-abstraction ==0.7.0.0, + any.th-compat ==0.1.5, + any.these ==1.2.1, + any.time ==1.12.2, + any.time-compat ==1.9.7, + any.transformers ==0.5.6.2, + any.transformers-compat ==0.7.2, + transformers-compat -five +five-three -four +generic-deriving +mtl -three -two, + any.unix ==2.7.3, + any.unordered-containers ==0.2.20, + unordered-containers -debug, + any.uuid-types ==1.0.6, + any.vector ==0.13.1.0, + vector +boundschecks -internalchecks -unsafechecks -wall, + any.vector-stream ==0.1.0.1, + any.witherable ==0.5 +index-state: hackage.haskell.org 2024-06-17T00:48:51Z From 3946cbd4a0b477700172c04be7baa3f43777bc99 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 24 Jun 2024 05:12:21 +0000 Subject: [PATCH 407/459] Upgrade docker build images --- build/README.md | 4 + build/darwin.aarch64/build | 1 - build/darwin.x86_64/build | 1 - build/linux.aarch64/Dockerfile | 2 +- build/linux.aarch64/build | 1 - build/linux.armv6hf/Dockerfile | 64 ++++++---------- build/linux.armv6hf/build | 3 +- build/linux.armv6hf/cabal.project.freeze | 93 ++++++++++++++++++++++++ build/linux.armv6hf/scutil | 48 ++++++++++++ build/linux.riscv64/Dockerfile | 49 +++++-------- build/linux.riscv64/build | 4 +- build/windows.x86_64/build | 1 - 12 files changed, 194 insertions(+), 77 deletions(-) create mode 100644 build/linux.armv6hf/cabal.project.freeze create mode 100644 build/linux.armv6hf/scutil diff --git a/build/README.md b/build/README.md index eb745a0..31e8607 100644 --- a/build/README.md +++ b/build/README.md @@ -11,3 +11,7 @@ This makes it simple to build any release without exotic hardware or software. An image can be built and tagged using `build_builder`, and run on a source tarball using `run_builder`. + +Tip: Are you developing an image that relies on QEmu usermode emulation? + It's easy to accidentally depend on binfmt\_misc on the host OS. + Do a `echo 0 | sudo tee /proc/sys/fs/binfmt_misc/status` before testing. diff --git a/build/darwin.aarch64/build b/build/darwin.aarch64/build index 4235d3a..ff522ff 100755 --- a/build/darwin.aarch64/build +++ b/build/darwin.aarch64/build @@ -4,7 +4,6 @@ set -xe tar xzv --strip-components=1 chmod +x striptests && ./striptests mkdir "$TARGETNAME" - cabal update ( IFS=';'; cabal build $CABALOPTS ) find . -name shellcheck -type f -exec mv {} "$TARGETNAME/" \; ls -l "$TARGETNAME" diff --git a/build/darwin.x86_64/build b/build/darwin.x86_64/build index 53857e8..058cece 100755 --- a/build/darwin.x86_64/build +++ b/build/darwin.x86_64/build @@ -4,7 +4,6 @@ set -xe tar xzv --strip-components=1 chmod +x striptests && ./striptests mkdir "$TARGETNAME" - cabal update ( IFS=';'; cabal build $CABALOPTS ) find . -name shellcheck -type f -exec mv {} "$TARGETNAME/" \; ls -l "$TARGETNAME" diff --git a/build/linux.aarch64/Dockerfile b/build/linux.aarch64/Dockerfile index fadb6a4..1ffe1bd 100644 --- a/build/linux.aarch64/Dockerfile +++ b/build/linux.aarch64/Dockerfile @@ -28,7 +28,7 @@ RUN curl -L "https://downloads.haskell.org/~cabal/cabal-install-3.9.0.0/cabal-in # Due to an apparent cabal bug, we specify our options directly to cabal # It won't reuse caches if ghc-options are specified in ~/.cabal/config -ENV CABALOPTS "--ghc-options;-split-sections -optc-Os -optc-Wl,--gc-sections -optc-fPIC;--with-ghc=$TARGET-ghc;--with-hc-pkg=$TARGET-ghc-pkg" +ENV CABALOPTS "--ghc-options;-split-sections -optc-Os -optc-Wl,--gc-sections -optc-fPIC;--with-ghc=$TARGET-ghc;--with-hc-pkg=$TARGET-ghc-pkg;-c;hashable -arch-native" # Prebuild the dependencies RUN cabal update && IFS=';' && cabal install --dependencies-only $CABALOPTS ShellCheck diff --git a/build/linux.aarch64/build b/build/linux.aarch64/build index f8001aa..3ce61ce 100755 --- a/build/linux.aarch64/build +++ b/build/linux.aarch64/build @@ -4,7 +4,6 @@ set -xe tar xzv --strip-components=1 chmod +x striptests && ./striptests mkdir "$TARGETNAME" - cabal update ( IFS=';'; cabal build $CABALOPTS --enable-executable-static ) find . -name shellcheck -type f -exec mv {} "$TARGETNAME/" \; ls -l "$TARGETNAME" diff --git a/build/linux.armv6hf/Dockerfile b/build/linux.armv6hf/Dockerfile index f933dda..b4d4197 100644 --- a/build/linux.armv6hf/Dockerfile +++ b/build/linux.armv6hf/Dockerfile @@ -1,25 +1,7 @@ -# I've again spent days trying to get a working armv6hf compiler going. -# God only knows how many recompilations of GCC, GHC, libraries, and -# ShellCheck itself, has gone into it. -# -# I tried Debian's toolchain. I tried my custom one built according to -# RPi `gcc -v`. I tried GHC9, glibc, musl, registerised vs not, but -# nothing has yielded an armv6hf binary that does not immediately -# segfault on qemu-arm-static or the RPi itself. -# -# I then tried the same but with armv7hf. Same story. -# -# Emulating the entire userspace with balenalib again? Very strange build -# failures where programs would fail to execute with > ~100 arguments. -# -# Finally, creating our own appears to work when using a custom QEmu -# patched to follow execve calls. -# -# PS: $100 bounty for getting a RPi1 compatible static build going -# with cross-compilation, similar to what the aarch64 build does. -# +# This Docker file uses a custom QEmu fork with patches to follow execve +# to build all of ShellCheck emulated. -FROM ubuntu:20.04 +FROM ubuntu:24.04 ENV TARGETNAME linux.armv6hf @@ -27,34 +9,34 @@ ENV TARGETNAME linux.armv6hf USER root ENV DEBIAN_FRONTEND noninteractive RUN apt-get update -RUN apt-get install -y build-essential git ninja-build python3 pkg-config libglib2.0-dev libpixman-1-dev -WORKDIR /build -RUN git clone --depth 1 https://github.com/koalaman/qemu -RUN cd qemu && ./configure --static && cd build && ninja qemu-arm -RUN cp qemu/build/qemu-arm /build/qemu-arm-static +RUN apt-get install -y --no-install-recommends build-essential git ninja-build python3 pkg-config libglib2.0-dev libpixman-1-dev python3-setuptools ca-certificates debootstrap +WORKDIR /qemu +RUN git clone --depth 1 https://github.com/koalaman/qemu . +RUN ./configure --static --disable-werror && cd build && ninja qemu-arm ENV QEMU_EXECVE 1 +# Convenience utility +COPY scutil /bin/scutil +COPY scutil /chroot/bin/scutil +RUN chmod +x /bin/scutil /chroot/bin/scutil + # Set up an armv6 userspace WORKDIR / -RUN apt-get install -y debootstrap qemu-user-static -# We expect this to fail if the host doesn't have binfmt qemu support -RUN qemu-debootstrap --arch armhf bullseye pi http://mirrordirector.raspbian.org/raspbian || [ -e /pi/etc/issue ] -RUN cp /build/qemu-arm-static /pi/usr/bin/qemu-arm-static -RUN printf > /bin/pirun '%s\n' '#!/bin/sh' 'chroot /pi /usr/bin/qemu-arm-static /usr/bin/env "$@"' && chmod +x /bin/pirun -# If the debootstrap process didn't finish, continue it -RUN [ ! -e /pi/debootstrap ] || pirun '/debootstrap/debootstrap' --second-stage +RUN debootstrap --arch armhf --variant=minbase --foreign bookworm /chroot http://mirrordirector.raspbian.org/raspbian +RUN cp /qemu/build/qemu-arm /chroot/bin/qemu +RUN scutil emu /debootstrap/debootstrap --second-stage # Install deps in the chroot -RUN pirun apt-get update -RUN pirun apt-get install -y ghc cabal-install +RUN scutil emu apt-get update +RUN scutil emu apt-get install -y --no-install-recommends ghc cabal-install +RUN scutil emu cabal update # Finally we can build the current dependencies. This takes hours. ENV CABALOPTS "--ghc-options;-split-sections -optc-Os -optc-Wl,--gc-sections;--gcc-options;-Os -Wl,--gc-sections -ffunction-sections -fdata-sections" -RUN pirun cabal update -RUN IFS=";" && pirun cabal install --dependencies-only $CABALOPTS ShellCheck -RUN IFS=';' && pirun cabal install $CABALOPTS --lib fgl +# Generated with `cabal freeze --constraint 'hashable -arch-native'` +COPY cabal.project.freeze /chroot/etc +RUN IFS=";" && scutil install_from_freeze /chroot/etc/cabal.project.freeze emu cabal install $CABALOPTS # Copy the build script -WORKDIR /pi/scratch -COPY build /pi/usr/bin -ENTRYPOINT ["/bin/pirun", "/usr/bin/build"] +COPY build /chroot/bin +ENTRYPOINT ["/bin/scutil", "emu", "/bin/build"] diff --git a/build/linux.armv6hf/build b/build/linux.armv6hf/build index daa94d9..1d496ae 100755 --- a/build/linux.armv6hf/build +++ b/build/linux.armv6hf/build @@ -1,8 +1,9 @@ #!/bin/sh set -xe -cd /scratch +mkdir /scratch && cd /scratch { tar xzv --strip-components=1 + cp /etc/cabal.project.freeze . chmod +x striptests && ./striptests mkdir "$TARGETNAME" # This script does not cabal update because compiling anything new is slow diff --git a/build/linux.armv6hf/cabal.project.freeze b/build/linux.armv6hf/cabal.project.freeze new file mode 100644 index 0000000..183bcc6 --- /dev/null +++ b/build/linux.armv6hf/cabal.project.freeze @@ -0,0 +1,93 @@ +active-repositories: hackage.haskell.org:merge +constraints: any.Diff ==0.5, + any.OneTuple ==0.4.2, + any.QuickCheck ==2.14.3, + QuickCheck -old-random +templatehaskell, + any.StateVar ==1.2.2, + any.aeson ==2.2.3.0, + aeson +ordered-keymap, + any.array ==0.5.4.0, + any.assoc ==1.1.1, + assoc -tagged, + any.base ==4.15.1.0, + any.base-orphans ==0.9.2, + any.bifunctors ==5.6.2, + bifunctors +tagged, + any.binary ==0.8.8.0, + any.bytestring ==0.10.12.1, + any.character-ps ==0.1, + any.comonad ==5.0.8, + comonad +containers +distributive +indexed-traversable, + any.containers ==0.6.4.1, + any.contravariant ==1.5.5, + contravariant +semigroups +statevar +tagged, + any.data-array-byte ==0.1.0.1, + any.data-fix ==0.3.3, + any.deepseq ==1.4.5.0, + any.directory ==1.3.6.2, + any.distributive ==0.6.2.1, + distributive +semigroups +tagged, + any.dlist ==1.0, + dlist -werror, + any.exceptions ==0.10.4, + any.fgl ==5.8.2.0, + fgl +containers042, + any.filepath ==1.4.2.1, + any.foldable1-classes-compat ==0.1, + foldable1-classes-compat +tagged, + any.generically ==0.1.1, + any.ghc-bignum ==1.1, + any.ghc-boot-th ==9.0.2, + any.ghc-prim ==0.7.0, + any.hashable ==1.4.6.0, + hashable -arch-native +integer-gmp -random-initial-seed, + any.indexed-traversable ==0.1.4, + any.indexed-traversable-instances ==0.1.2, + any.integer-conversion ==0.1.1, + any.integer-logarithms ==1.0.3.1, + integer-logarithms -check-bounds +integer-gmp, + any.mtl ==2.2.2, + any.network-uri ==2.6.4.2, + any.parsec ==3.1.14.0, + any.pretty ==1.1.3.6, + any.primitive ==0.9.0.0, + any.process ==1.6.13.2, + any.random ==1.2.1.2, + any.regex-base ==0.94.0.2, + any.regex-tdfa ==1.3.2.2, + regex-tdfa +doctest -force-o2, + any.rts ==1.0.2, + any.scientific ==0.3.8.0, + scientific -integer-simple, + any.semialign ==1.3.1, + semialign +semigroupoids, + any.semigroupoids ==6.0.1, + semigroupoids +comonad +containers +contravariant +distributive +tagged +unordered-containers, + any.splitmix ==0.1.0.5, + splitmix -optimised-mixer, + any.stm ==2.5.0.0, + any.strict ==0.5, + any.tagged ==0.8.8, + tagged +deepseq +transformers, + any.template-haskell ==2.17.0.0, + any.text ==1.2.5.0, + any.text-iso8601 ==0.1.1, + any.text-short ==0.1.6, + text-short -asserts, + any.th-abstraction ==0.7.0.0, + any.th-compat ==0.1.5, + any.these ==1.2.1, + any.time ==1.9.3, + any.time-compat ==1.9.7, + any.transformers ==0.5.6.2, + any.transformers-compat ==0.7.2, + transformers-compat -five +five-three -four +generic-deriving +mtl -three -two, + any.unix ==2.7.2.2, + any.unordered-containers ==0.2.20, + unordered-containers -debug, + any.uuid-types ==1.0.6, + any.vector ==0.13.1.0, + vector +boundschecks -internalchecks -unsafechecks -wall, + any.vector-stream ==0.1.0.1, + any.witherable ==0.5 +index-state: hackage.haskell.org 2024-06-18T02:21:19Z diff --git a/build/linux.armv6hf/scutil b/build/linux.armv6hf/scutil new file mode 100644 index 0000000..a85d810 --- /dev/null +++ b/build/linux.armv6hf/scutil @@ -0,0 +1,48 @@ +#!/bin/dash +# Various ShellCheck build utility functions + +# Generally set a ulimit to avoid QEmu using too much memory +ulimit -v "$((10*1024*1024))" +# If we happen to invoke or run under QEmu, make sure to follow execve. +# This requires a patched QEmu. +export QEMU_EXECVE=1 + +# Retry a command until it succeeds +# Usage: scutil retry 3 mycmd +retry() { + n="$1" + ret=1 + shift + while [ "$n" -gt 0 ] + do + "$@" + ret=$? + [ "$ret" = 0 ] && break + n=$((n-1)) + done + return "$ret" +} + +# Install all dependencies from a freeze file +# Usage: scutil install_from_freeze /path/cabal.project.freeze cabal install +install_from_freeze() { + linefeed=$(printf '\nx') + linefeed=${linefeed%x} + flags=$( + sed 's/constraints:/&\n /' "$1" | + grep -vw -e rts -e base | + sed -n -e 's/^ *\([^,]*\).*/\1/p' | + sed -e 's/any\.\([^ ]*\) ==\(.*\)/\1-\2/; te; s/.*/--constraint\n&/; :e') + shift + # shellcheck disable=SC2086 + ( IFS=$linefeed; set -x; "$@" $flags ) +} + +# Run a command under emulation. +# This assumes the correct emulator is named 'qemu' and the chroot is /chroot +# Usage: scutil emu echo "Hello World" +emu() { + chroot /chroot /bin/qemu /usr/bin/env "$@" +} + +"$@" diff --git a/build/linux.riscv64/Dockerfile b/build/linux.riscv64/Dockerfile index 0ac95e5..d138ff7 100644 --- a/build/linux.riscv64/Dockerfile +++ b/build/linux.riscv64/Dockerfile @@ -10,46 +10,37 @@ ENV DEBIAN_FRONTEND noninteractive RUN apt-get update -y # Install qemu -RUN apt-get install -y --no-install-recommends build-essential ninja-build python3 pkg-config libglib2.0-dev libpixman-1-dev curl ca-certificates python3-virtualenv git python3-setuptools +RUN apt-get install -y --no-install-recommends build-essential ninja-build python3 pkg-config libglib2.0-dev libpixman-1-dev curl ca-certificates python3-virtualenv git python3-setuptools debootstrap WORKDIR /qemu RUN git clone --depth 1 https://github.com/koalaman/qemu . -#RUN git clone https://github.com/balena-io/qemu . -# Release 7.0.0 -#RUN git checkout 639d1d8903f65d74eb04c49e0df7a4b2f014cd86 RUN ./configure --target-list=riscv64-linux-user --static --disable-system --disable-pie --disable-werror RUN cd build && ninja qemu-riscv64 ENV QEMU_EXECVE 1 +# Convenience utility +COPY scutil /bin/scutil +# We have to copy to /usr/bin because debootstrap will try to symlink /bin and fail if it exists +COPY scutil /chroot/usr/bin/scutil +RUN chmod +x /bin/scutil /chroot/usr/bin/scutil # Set up a riscv64 userspace -RUN apt-get install -y --no-install-recommends debootstrap -RUN debootstrap --arch=riscv64 --foreign noble /rvfs http://ports.ubuntu.com/ubuntu-ports -RUN cp /qemu/build/qemu-riscv64 /rvfs/usr/bin/qemu-riscv64-static - -# Command to run riscv binaries in the chroot. The Haskell runtime allocates 1TB -# vspace up front and QEmu has a RAM cost per vspace, so use ulimit to allocate -# less and reduce RAM usage. -RUN printf > /bin/rv '%s\n' '#!/bin/sh' 'ulimit -v $((10*1024*1024)); chroot /rvfs /usr/bin/qemu-riscv64-static /usr/bin/env "$@"' -RUN chmod +x /bin/rv -RUN [ ! -e /rvfs/debootstrap ] || rv '/debootstrap/debootstrap' --second-stage +WORKDIR / +RUN debootstrap --arch=riscv64 --variant=minbase --components=main,universe --foreign noble /chroot http://ports.ubuntu.com/ubuntu-ports +RUN cp /qemu/build/qemu-riscv64 /chroot/bin/qemu +RUN scutil emu /debootstrap/debootstrap --second-stage # Install deps in the chroot -RUN printf > /rvfs/etc/apt/sources.list '%s\n' 'deb http://ports.ubuntu.com/ubuntu-ports noble main universe' -RUN rv apt-get update -y -RUN rv apt-get install -y --no-install-recommends ghc cabal-install +RUN scutil emu apt-get update +RUN scutil emu apt-get install -y --no-install-recommends ghc cabal-install +RUN scutil emu cabal update -RUN rv cabal update # Generated with: cabal freeze -c 'hashable -arch-native'. We put it in /etc so cabal won't find it. -COPY cabal.project.freeze /rvfs/etc -# Awful hack to install everything from the freeze file -# This basically turns 'any.tagged ==0.8.8' into tagged-0.8.8 to install by version, -# and adds a -c before 'hashable -arch-native +integer-gmp' to make it a flag constraint. -RUN < /rvfs/etc/cabal.project.freeze sed 's/constraints:/&\n /' | grep -vw rts | sed -n -e 's/^ *\([^,]*\).*/\1/p' | sed -e 's/any\.\([^ ]*\) ==\(.*\)/\1-\2/; te; s/.*/-c\n&/; :e' > /tmp/preinstall-flags -# Finally we can build the current dependencies. This takes hours. -# There's apparently a random segfault during assembly, so retry a few times. -RUN set -x; IFS=${IFS# }; f() { rv cabal install --keep-going $(cat /tmp/preinstall-flags); }; for i in $(seq 5); do f; ret=$?; [ $ret = 0 ] && break; done; exit $ret +COPY cabal.project.freeze /chroot/etc + +# Build all dependencies from the freeze file. The emulator segfaults at random, +# so retry a few times. +RUN scutil install_from_freeze /chroot/etc/cabal.project.freeze retry 5 emu cabal install --keep-going # Copy the build script -WORKDIR /rvfs/scratch -COPY build /rvfs/usr/bin/build -ENTRYPOINT ["/bin/rv", "/usr/bin/build"] +COPY build /chroot/bin/build +ENTRYPOINT ["/bin/scutil", "emu", "/bin/build"] diff --git a/build/linux.riscv64/build b/build/linux.riscv64/build index 63c487f..ed9dc27 100755 --- a/build/linux.riscv64/build +++ b/build/linux.riscv64/build @@ -2,6 +2,8 @@ set -xe IFS=';' { + mkdir -p /tmp/scratch + cd /tmp/scratch tar xzv --strip-components=1 chmod +x striptests && ./striptests # Use a freeze file to ensure we use the same dependencies we cached during @@ -9,7 +11,7 @@ IFS=';' cp /etc/cabal.project.freeze . mkdir "$TARGETNAME" # Retry in case of random segfault - cabal build --enable-executable-static || cabal build --enable-executable-static + scutil retry 3 cabal build --enable-executable-static find . -name shellcheck -type f -exec mv {} "$TARGETNAME/" \; ls -l "$TARGETNAME" "$TARGET-strip" -s "$TARGETNAME/shellcheck" diff --git a/build/windows.x86_64/build b/build/windows.x86_64/build index 7bf186e..22e5b42 100755 --- a/build/windows.x86_64/build +++ b/build/windows.x86_64/build @@ -8,7 +8,6 @@ set -xe tar xzv --strip-components=1 chmod +x striptests && ./striptests mkdir "$TARGETNAME" - cabal update ( IFS=';'; cabal build $CABALOPTS ) find dist*/ -name shellcheck.exe -type f -ls -exec mv {} "$TARGETNAME/" \; ls -l "$TARGETNAME" From b408f546209bdd344177d2d778b38d3a77f0db78 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 7 Jul 2024 01:03:40 -0400 Subject: [PATCH 408/459] Simplify invokedNodes --- src/ShellCheck/CFGAnalysis.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/CFGAnalysis.hs b/src/ShellCheck/CFGAnalysis.hs index 0b99c9f..8534d6f 100644 --- a/src/ShellCheck/CFGAnalysis.hs +++ b/src/ShellCheck/CFGAnalysis.hs @@ -1350,7 +1350,7 @@ analyzeControlFlow params t = -- All nodes we've touched invocations <- readSTRef $ cInvocations ctx - let invokedNodes = M.fromDistinctAscList $ map (\c -> (c, ())) $ S.toList $ M.keysSet $ groupByNode $ M.map snd invocations + let invokedNodes = M.fromSet (const ()) $ S.unions $ map (M.keysSet . snd) $ M.elems invocations -- Invoke all functions that were declared but not invoked -- This is so that we still get warnings for dead code From 61b7e66f809d9c2d1be06e5787ae6a98039e798e Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 7 Jul 2024 01:07:39 -0400 Subject: [PATCH 409/459] Use sets instead of maps that never use their values --- src/ShellCheck/Analytics.hs | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index a89f940..621f70a 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -491,14 +491,14 @@ checkWrongArithmeticAssignment params (T_SimpleCommand id [T_Assignment _ _ _ _ sequence_ $ do str <- getNormalString val var:op:_ <- matchRegex regex str - Map.lookup var references + guard $ S.member var references return . warn (getId val) 2100 $ "Use $((..)) for arithmetics, e.g. i=$((i " ++ op ++ " 2))" where regex = mkRegex "^([_a-zA-Z][_a-zA-Z0-9]*)([+*-]).+$" - references = foldl (flip ($)) Map.empty (map insertRef $ variableFlow params) + references = foldl (flip ($)) S.empty (map insertRef $ variableFlow params) insertRef (Assignment (_, _, name, _)) = - Map.insert name () + S.insert name insertRef _ = Prelude.id getNormalString (T_NormalWord _ words) = do @@ -974,32 +974,32 @@ prop_checkArrayWithoutIndex9 = verifyTree checkArrayWithoutIndex "read -r -a arr prop_checkArrayWithoutIndex10 = verifyTree checkArrayWithoutIndex "read -ra arr <<< 'foo bar'; echo \"$arr\"" prop_checkArrayWithoutIndex11 = verifyNotTree checkArrayWithoutIndex "read -rpfoobar r; r=42" checkArrayWithoutIndex params _ = - doVariableFlowAnalysis readF writeF defaultMap (variableFlow params) + doVariableFlowAnalysis readF writeF defaultSet (variableFlow params) where - defaultMap = Map.fromList $ map (\x -> (x,())) arrayVariables + defaultSet = S.fromList arrayVariables readF _ (T_DollarBraced id _ token) _ = do - map <- get + s <- get return . maybeToList $ do name <- getLiteralString token - assigned <- Map.lookup name map + guard $ S.member name s return $ makeComment WarningC id 2128 "Expanding an array without an index only gives the first element." readF _ _ _ = return [] writeF _ (T_Assignment id mode name [] _) _ (DataString _) = do - isArray <- gets (Map.member name) + isArray <- gets (S.member name) return $ if not isArray then [] else case mode of Assign -> [makeComment WarningC id 2178 "Variable was used as an array but is now assigned a string."] Append -> [makeComment WarningC id 2179 "Use array+=(\"item\") to append items to an array."] writeF _ t name (DataArray _) = do - modify (Map.insert name ()) + modify (S.insert name) return [] writeF _ expr name _ = do if isIndexed expr - then modify (Map.insert name ()) - else modify (Map.delete name) + then modify (S.insert name) + else modify (S.delete name) return [] isIndexed expr = @@ -3968,12 +3968,12 @@ prop_checkTranslatedStringVariable4 = verifyNot checkTranslatedStringVariable "v prop_checkTranslatedStringVariable5 = verifyNot checkTranslatedStringVariable "foo=var; bar=val2; $\"foo bar\"" checkTranslatedStringVariable params (T_DollarDoubleQuoted id [T_Literal _ s]) | all isVariableChar s - && Map.member s assignments + && S.member s assignments = warnWithFix id 2256 "This translated string is the name of a variable. Flip leading $ and \" if this should be a quoted substitution." (fix id) where - assignments = foldl (flip ($)) Map.empty (map insertAssignment $ variableFlow params) - insertAssignment (Assignment (_, token, name, _)) | isVariableName name = - Map.insert name token + assignments = foldl (flip ($)) S.empty (map insertAssignment $ variableFlow params) + insertAssignment (Assignment (_, _, name, _)) | isVariableName name = + S.insert name insertAssignment _ = Prelude.id fix id = fixWith [replaceStart id params 2 "\"$"] checkTranslatedStringVariable _ _ = return () From 8746c6e7f20fdaa2463ae008dc527375ab76b04e Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 7 Jul 2024 01:07:53 -0400 Subject: [PATCH 410/459] Switch the order of the maps to avoid unnecessary unionWith instead of union --- src/ShellCheck/CFGAnalysis.hs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ShellCheck/CFGAnalysis.hs b/src/ShellCheck/CFGAnalysis.hs index 8534d6f..27098b1 100644 --- a/src/ShellCheck/CFGAnalysis.hs +++ b/src/ShellCheck/CFGAnalysis.hs @@ -133,7 +133,7 @@ internalToExternal s = literalValue = Nothing } } - flatVars = M.unionsWith (\_ last -> last) $ map mapStorage [sGlobalValues s, sLocalValues s, sPrefixValues s] + flatVars = M.unions $ map mapStorage [sPrefixValues s, sLocalValues s, sGlobalValues s] -- Conveniently get the state before a token id getIncomingState :: CFGAnalysis -> Id -> Maybe ProgramState @@ -672,7 +672,7 @@ vmPatch base diff = _ | vmIsQuickEqual base diff -> diff _ -> VersionedMap { mapVersion = -1, - mapStorage = M.unionWith (flip const) (mapStorage base) (mapStorage diff) + mapStorage = M.union (mapStorage diff) (mapStorage base) } -- Set a variable. This includes properties. Applies it to the appropriate scope. @@ -1373,7 +1373,7 @@ analyzeControlFlow params t = -- Fill in the map with unreachable states for anything we didn't get to let baseStates = M.fromDistinctAscList $ map (\c -> (c, (unreachableState, unreachableState))) $ uncurry enumFromTo $ nodeRange $ cfGraph cfg - let allStates = M.unionWith (flip const) baseStates invokedStates + let allStates = M.union invokedStates baseStates -- Convert to external states let nodeToData = M.map (\(a,b) -> (internalToExternal a, internalToExternal b)) allStates From e5fdec970ae3bc16369b3ec289a4c4fbb9254751 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 7 Jul 2024 01:08:18 -0400 Subject: [PATCH 411/459] Swap the order of the tuple returned by orderEdge --- src/ShellCheck/CFG.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/CFG.hs b/src/ShellCheck/CFG.hs index e1d3259..a720911 100644 --- a/src/ShellCheck/CFG.hs +++ b/src/ShellCheck/CFG.hs @@ -300,13 +300,13 @@ removeUnnecessaryStructuralNodes (nodes, edges, mapping, association) = edgesToCollapse = S.fromList $ filter filterEdges regularEdges remapping :: M.Map Node Node - remapping = foldl' (\m (new, old) -> M.insert old new m) M.empty $ map orderEdge $ S.toList edgesToCollapse + remapping = foldl' (\m (old, new) -> M.insert old new m) M.empty $ map orderEdge $ S.toList edgesToCollapse recursiveRemapping = M.fromList $ map (\c -> (c, recursiveLookup remapping c)) $ M.keys remapping filterEdges (a,b,_) = a `S.member` candidateNodes && b `S.member` candidateNodes - orderEdge (a,b,_) = if a < b then (a,b) else (b,a) + orderEdge (a,b,_) = if a < b then (b,a) else (a,b) counter = foldl' (\map key -> M.insertWith (+) key 1 map) M.empty isRegularEdge (_, _, CFEFlow) = True isRegularEdge _ = False From 95c0cc2e4bdbad193ba3616a5ecfd2741b1f3807 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 7 Jul 2024 01:09:17 -0400 Subject: [PATCH 412/459] Simplify removeUnnecessaryStructuralNodes --- src/ShellCheck/CFG.hs | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/src/ShellCheck/CFG.hs b/src/ShellCheck/CFG.hs index a720911..dc56b58 100644 --- a/src/ShellCheck/CFG.hs +++ b/src/ShellCheck/CFG.hs @@ -295,19 +295,19 @@ removeUnnecessaryStructuralNodes (nodes, edges, mapping, association) = regularEdges = filter isRegularEdge edges inDegree = counter $ map (\(from,to,_) -> from) regularEdges outDegree = counter $ map (\(from,to,_) -> to) regularEdges - structuralNodes = S.fromList $ map fst $ filter isStructural nodes + structuralNodes = S.fromList [node | (node, CFStructuralNode) <- nodes] candidateNodes = S.filter isLinear structuralNodes edgesToCollapse = S.fromList $ filter filterEdges regularEdges remapping :: M.Map Node Node - remapping = foldl' (\m (old, new) -> M.insert old new m) M.empty $ map orderEdge $ S.toList edgesToCollapse - recursiveRemapping = M.fromList $ map (\c -> (c, recursiveLookup remapping c)) $ M.keys remapping + remapping = M.fromList $ map orderEdge $ S.toList edgesToCollapse + recursiveRemapping = M.mapWithKey (\c _ -> recursiveLookup remapping c) remapping filterEdges (a,b,_) = a `S.member` candidateNodes && b `S.member` candidateNodes orderEdge (a,b,_) = if a < b then (b,a) else (a,b) - counter = foldl' (\map key -> M.insertWith (+) key 1 map) M.empty + counter = M.fromListWith (+) . map (\key -> (key, 1)) isRegularEdge (_, _, CFEFlow) = True isRegularEdge _ = False @@ -317,11 +317,6 @@ removeUnnecessaryStructuralNodes (nodes, edges, mapping, association) = Nothing -> node Just x -> recursiveLookup map x - isStructural (node, label) = - case label of - CFStructuralNode -> True - _ -> False - isLinear node = M.findWithDefault 0 node inDegree == 1 && M.findWithDefault 0 node outDegree == 1 From 98b8dc0720148d69ff924f89966c50dc2dda2fe3 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Sun, 7 Jul 2024 01:11:00 -0400 Subject: [PATCH 413/459] Use fromList instead of reimplementing it in terms of foldl --- src/ShellCheck/Analytics.hs | 20 ++++---------------- src/ShellCheck/CFGAnalysis.hs | 2 +- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 621f70a..211993c 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -496,10 +496,7 @@ checkWrongArithmeticAssignment params (T_SimpleCommand id [T_Assignment _ _ _ _ "Use $((..)) for arithmetics, e.g. i=$((i " ++ op ++ " 2))" where regex = mkRegex "^([_a-zA-Z][_a-zA-Z0-9]*)([+*-]).+$" - references = foldl (flip ($)) S.empty (map insertRef $ variableFlow params) - insertRef (Assignment (_, _, name, _)) = - S.insert name - insertRef _ = Prelude.id + references = S.fromList [name | Assignment (_, _, name, _) <- variableFlow params] getNormalString (T_NormalWord _ words) = do parts <- mapM getLiterals words @@ -2380,15 +2377,9 @@ prop_checkUnused51 = verifyTree checkUnusedAssignments "x[y[z=1]]=1; echo ${x[@] checkUnusedAssignments params t = execWriter (mapM_ warnFor unused) where flow = variableFlow params - references = foldl (flip ($)) defaultMap (map insertRef flow) - insertRef (Reference (base, token, name)) = - Map.insert (stripSuffix name) () - insertRef _ = id + references = Map.union (Map.fromList [(stripSuffix name, ()) | Reference (base, token, name) <- flow]) defaultMap - assignments = foldl (flip ($)) Map.empty (map insertAssignment flow) - insertAssignment (Assignment (_, token, name, _)) | isVariableName name = - Map.insert name token - insertAssignment _ = id + assignments = Map.fromList [(name, token) | Assignment (_, token, name, _) <- flow, isVariableName name] unused = Map.assocs $ Map.difference assignments references @@ -3971,10 +3962,7 @@ checkTranslatedStringVariable params (T_DollarDoubleQuoted id [T_Literal _ s]) && S.member s assignments = warnWithFix id 2256 "This translated string is the name of a variable. Flip leading $ and \" if this should be a quoted substitution." (fix id) where - assignments = foldl (flip ($)) S.empty (map insertAssignment $ variableFlow params) - insertAssignment (Assignment (_, _, name, _)) | isVariableName name = - S.insert name - insertAssignment _ = Prelude.id + assignments = S.fromList [name | Assignment (_, _, name, _) <- variableFlow params, isVariableName name] fix id = fixWith [replaceStart id params 2 "\"$"] checkTranslatedStringVariable _ _ = return () diff --git a/src/ShellCheck/CFGAnalysis.hs b/src/ShellCheck/CFGAnalysis.hs index 27098b1..cf982e0 100644 --- a/src/ShellCheck/CFGAnalysis.hs +++ b/src/ShellCheck/CFGAnalysis.hs @@ -1286,7 +1286,7 @@ dataflow ctx entry = do else do let (next, rest) = S.deleteFindMin ps nexts <- process states next - writeSTRef pending $ foldl (flip S.insert) rest nexts + writeSTRef pending $ S.union (S.fromList nexts) rest f (n-1) pending states process states node = do From 6593096ba06e6b54ec08d00a9c625930a357cdfe Mon Sep 17 00:00:00 2001 From: Sertonix Date: Fri, 28 Jun 2024 16:24:06 +0200 Subject: [PATCH 414/459] Allow SC3003 on busybox shell --- src/ShellCheck/Checks/ShellSupport.hs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index cab0546..9192c0e 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -212,6 +212,8 @@ prop_checkBashisms118 = verify checkBashisms "#!/bin/busybox sh\nxyz=1\n${!x*}" prop_checkBashisms119 = verify checkBashisms "#!/bin/busybox sh\nx='test'\n${x^^[t]}" -- SC3059 prop_checkBashisms120 = verify checkBashisms "#!/bin/sh\n[ x == y ]" prop_checkBashisms121 = verifyNot checkBashisms "#!/bin/sh\n# shellcheck shell=busybox\n[ x == y ]" +prop_checkBashisms122 = verify checkBashisms "#!/bin/dash\n$'a'" +prop_checkBashisms123 = verifyNot checkBashisms "#!/bin/busybox sh\n$'a'" checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do params <- ask kludge params t @@ -229,7 +231,8 @@ checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do bashism (T_ProcSub id _ _) = warnMsg id 3001 "process substitution is" bashism (T_Extglob id _ _) = warnMsg id 3002 "extglob is" - bashism (T_DollarSingleQuoted id _) = warnMsg id 3003 "$'..' is" + bashism (T_DollarSingleQuoted id _) = + unless isBusyboxSh $ warnMsg id 3003 "$'..' is" bashism (T_DollarDoubleQuoted id _) = warnMsg id 3004 "$\"..\" is" bashism (T_ForArithmetic id _ _ _ _) = warnMsg id 3005 "arithmetic for loops are" bashism (T_Arithmetic id _) = warnMsg id 3006 "standalone ((..)) is" From 4c852749214e102f7a1daec08094ed91b08867f7 Mon Sep 17 00:00:00 2001 From: Sertonix Date: Tue, 2 Jul 2024 17:06:35 +0200 Subject: [PATCH 415/459] Fix SC3045 for busybox shell --- src/ShellCheck/Checks/ShellSupport.hs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 9192c0e..1207ad0 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -214,6 +214,9 @@ prop_checkBashisms120 = verify checkBashisms "#!/bin/sh\n[ x == y ]" prop_checkBashisms121 = verifyNot checkBashisms "#!/bin/sh\n# shellcheck shell=busybox\n[ x == y ]" prop_checkBashisms122 = verify checkBashisms "#!/bin/dash\n$'a'" prop_checkBashisms123 = verifyNot checkBashisms "#!/bin/busybox sh\n$'a'" +prop_checkBashisms124 = verify checkBashisms "#!/bin/dash\ntype -p test" +prop_checkBashisms125 = verifyNot checkBashisms "#!/bin/busybox sh\ntype -p test" +prop_checkBashisms126 = verifyNot checkBashisms "#!/bin/busybox sh\nread -p foo -r bar" checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do params <- ask kludge params t @@ -446,10 +449,10 @@ checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do ("hash", Just $ if isDash then ["r", "v"] else ["r"]), ("jobs", Just ["l", "p"]), ("printf", Just []), - ("read", Just $ if isDash then ["r", "p"] else ["r"]), + ("read", Just $ if isDash || isBusyboxSh then ["r", "p"] else ["r"]), ("readonly", Just ["p"]), ("trap", Just []), - ("type", Just []), + ("type", Just $ if isBusyboxSh then ["p"] else []), ("ulimit", if isDash then Nothing else Just ["f"]), ("umask", Just ["S"]), ("unset", Just ["f", "v"]), From 6d2f3d8628235dd2146cd248d9828b004852c7ad Mon Sep 17 00:00:00 2001 From: Sertonix Date: Tue, 9 Jul 2024 15:12:16 +0200 Subject: [PATCH 416/459] Allow 'echo -e' in busybox shell --- src/ShellCheck/Checks/ShellSupport.hs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 1207ad0..2ec4cf7 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -217,6 +217,7 @@ prop_checkBashisms123 = verifyNot checkBashisms "#!/bin/busybox sh\n$'a'" prop_checkBashisms124 = verify checkBashisms "#!/bin/dash\ntype -p test" prop_checkBashisms125 = verifyNot checkBashisms "#!/bin/busybox sh\ntype -p test" prop_checkBashisms126 = verifyNot checkBashisms "#!/bin/busybox sh\nread -p foo -r bar" +prop_checkBashisms127 = verifyNot checkBashisms "#!/bin/busybox sh\necho -ne foo" checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do params <- ask kludge params t @@ -327,7 +328,11 @@ checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do bashism t@(T_SimpleCommand _ _ (cmd:arg:_)) | t `isCommand` "echo" && argString `matches` flagRegex = - if isDash + if isBusyboxSh + then + when (not (argString `matches` busyboxFlagRegex)) $ + warnMsg (getId arg) 3036 "echo flags besides -n and -e" + else if isDash then when (argString /= "-n") $ warnMsg (getId arg) 3036 "echo flags besides -n" @@ -336,6 +341,7 @@ checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do where argString = concat $ oversimplify arg flagRegex = mkRegex "^-[eEsn]+$" + busyboxFlagRegex = mkRegex "^-[en]+$" bashism t@(T_SimpleCommand _ _ (cmd:arg:_)) | getLiteralString cmd == Just "exec" && "-" `isPrefixOf` concat (oversimplify arg) = From d590a35ff8093a3a1edeac67cdb7fb9cffa593bf Mon Sep 17 00:00:00 2001 From: Hasit Mistry Date: Tue, 9 Jul 2024 14:22:19 -0700 Subject: [PATCH 417/459] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 366c427..03d4d4a 100644 --- a/README.md +++ b/README.md @@ -113,6 +113,7 @@ Services and platforms that have ShellCheck pre-installed and ready to use: * [CircleCI](https://circleci.com) via the [ShellCheck Orb](https://circleci.com/orbs/registry/orb/circleci/shellcheck) * [Github](https://github.com/features/actions) (only Linux) * [Trunk Check](https://trunk.io/products/check) (universal linter; [allows you to explicitly version your shellcheck install](https://github.com/trunk-io/plugins/blob/bcbb361dcdbe4619af51ea7db474d7fb87540d20/.trunk/trunk.yaml#L32)) via the [shellcheck plugin](https://github.com/trunk-io/plugins/blob/main/linters/shellcheck/plugin.yaml) +* [CodeRabbit](https://coderabbit.ai/) Most other services, including [GitLab](https://about.gitlab.com/), let you install ShellCheck yourself, either through the system's package manager (see [Installing](#installing)), From 2696c6472dd55880f4dc1350262e1192953754ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Santoro?= Date: Wed, 31 Jul 2024 12:52:42 +0000 Subject: [PATCH 418/459] Whitelist oc to avoid SC2016 false positive Fixes #3033. --- src/ShellCheck/Analytics.hs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 211993c..08eed57 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1097,6 +1097,7 @@ checkSingleQuotedVariables params t@(T_SingleQuoted id s) = ,"sudo" -- covering "sudo sh" and such ,"docker" -- like above ,"podman" + ,"oc" ,"dpkg-query" ,"jq" -- could also check that user provides --arg ,"rename" From 38c5ba7c79e35af29bb1496e774af1d5add0e73c Mon Sep 17 00:00:00 2001 From: Emil Berg Date: Sat, 3 Aug 2024 08:49:40 +0200 Subject: [PATCH 419/459] Fix typos and trailing whitespace --- .github/ISSUE_TEMPLATE.md | 4 ++-- README.md | 2 +- src/ShellCheck/AST.hs | 4 ++-- src/ShellCheck/Analytics.hs | 6 +++--- src/ShellCheck/CFG.hs | 2 +- src/ShellCheck/Checks/Custom.hs | 2 +- src/ShellCheck/Parser.hs | 2 +- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 44d151e..493b465 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -1,6 +1,6 @@ #### For bugs -- Rule Id (if any, e.g. SC1000): -- My shellcheck version (`shellcheck --version` or "online"): +- Rule Id (if any, e.g. SC1000): +- My shellcheck version (`shellcheck --version` or "online"): - [ ] The rule's wiki page does not already cover this (e.g. https://shellcheck.net/wiki/SC2086) - [ ] I tried on https://www.shellcheck.net/ and verified that this is still a problem on the latest commit diff --git a/README.md b/README.md index 366c427..78802a0 100644 --- a/README.md +++ b/README.md @@ -233,7 +233,7 @@ Alternatively, you can download pre-compiled binaries for the latest release her * [Linux, x86_64](https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.x86_64.tar.xz) (statically linked) * [Linux, armv6hf](https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.armv6hf.tar.xz), i.e. Raspberry Pi (statically linked) * [Linux, aarch64](https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.aarch64.tar.xz) aka ARM64 (statically linked) -* [macOS, aarch64](https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.darwin.aarch64.tar.xz) +* [macOS, aarch64](https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.darwin.aarch64.tar.xz) * [macOS, x86_64](https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.darwin.x86_64.tar.xz) * [Windows, x86](https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.zip) diff --git a/src/ShellCheck/AST.hs b/src/ShellCheck/AST.hs index ca05c98..979a9b0 100644 --- a/src/ShellCheck/AST.hs +++ b/src/ShellCheck/AST.hs @@ -206,7 +206,7 @@ pattern T_Annotation id anns t = OuterToken id (Inner_T_Annotation anns t) pattern T_Arithmetic id c = OuterToken id (Inner_T_Arithmetic c) pattern T_Array id t = OuterToken id (Inner_T_Array t) pattern TA_Sequence id l = OuterToken id (Inner_TA_Sequence l) -pattern TA_Parentesis id t = OuterToken id (Inner_TA_Parenthesis t) +pattern TA_Parenthesis id t = OuterToken id (Inner_TA_Parenthesis t) pattern T_Assignment id mode var indices value = OuterToken id (Inner_T_Assignment mode var indices value) pattern TA_Trinary id t1 t2 t3 = OuterToken id (Inner_TA_Trinary t1 t2 t3) pattern TA_Unary id op t1 = OuterToken id (Inner_TA_Unary op t1) @@ -259,7 +259,7 @@ pattern T_Subshell id l = OuterToken id (Inner_T_Subshell l) pattern T_UntilExpression id c l = OuterToken id (Inner_T_UntilExpression c l) pattern T_WhileExpression id c l = OuterToken id (Inner_T_WhileExpression c l) -{-# COMPLETE T_AND_IF, T_Bang, T_Case, TC_Empty, T_CLOBBER, T_DGREAT, T_DLESS, T_DLESSDASH, T_Do, T_DollarSingleQuoted, T_Done, T_DSEMI, T_Elif, T_Else, T_EOF, T_Esac, T_Fi, T_For, T_Glob, T_GREATAND, T_Greater, T_If, T_In, T_Lbrace, T_Less, T_LESSAND, T_LESSGREAT, T_Literal, T_Lparen, T_NEWLINE, T_OR_IF, T_ParamSubSpecialChar, T_Pipe, T_Rbrace, T_Rparen, T_Select, T_Semi, T_SingleQuoted, T_Then, T_UnparsedIndex, T_Until, T_While, TA_Assignment, TA_Binary, TA_Expansion, T_AndIf, T_Annotation, T_Arithmetic, T_Array, TA_Sequence, TA_Parentesis, T_Assignment, TA_Trinary, TA_Unary, TA_Variable, T_Backgrounded, T_Backticked, T_Banged, T_BatsTest, T_BraceExpansion, T_BraceGroup, TC_And, T_CaseExpression, TC_Binary, TC_Group, TC_Nullary, T_Condition, T_CoProcBody, T_CoProc, TC_Or, TC_Unary, T_DollarArithmetic, T_DollarBraceCommandExpansion, T_DollarBraced, T_DollarBracket, T_DollarDoubleQuoted, T_DollarExpansion, T_DoubleQuoted, T_Extglob, T_FdRedirect, T_ForArithmetic, T_ForIn, T_Function, T_HereDoc, T_HereString, T_IfExpression, T_Include, T_IndexedElement, T_IoDuplicate, T_IoFile, T_NormalWord, T_OrIf, T_Pipeline, T_ProcSub, T_Redirecting, T_Script, T_SelectIn, T_SimpleCommand, T_SourceCommand, T_Subshell, T_UntilExpression, T_WhileExpression #-} +{-# COMPLETE T_AND_IF, T_Bang, T_Case, TC_Empty, T_CLOBBER, T_DGREAT, T_DLESS, T_DLESSDASH, T_Do, T_DollarSingleQuoted, T_Done, T_DSEMI, T_Elif, T_Else, T_EOF, T_Esac, T_Fi, T_For, T_Glob, T_GREATAND, T_Greater, T_If, T_In, T_Lbrace, T_Less, T_LESSAND, T_LESSGREAT, T_Literal, T_Lparen, T_NEWLINE, T_OR_IF, T_ParamSubSpecialChar, T_Pipe, T_Rbrace, T_Rparen, T_Select, T_Semi, T_SingleQuoted, T_Then, T_UnparsedIndex, T_Until, T_While, TA_Assignment, TA_Binary, TA_Expansion, T_AndIf, T_Annotation, T_Arithmetic, T_Array, TA_Sequence, TA_Parenthesis, T_Assignment, TA_Trinary, TA_Unary, TA_Variable, T_Backgrounded, T_Backticked, T_Banged, T_BatsTest, T_BraceExpansion, T_BraceGroup, TC_And, T_CaseExpression, TC_Binary, TC_Group, TC_Nullary, T_Condition, T_CoProcBody, T_CoProc, TC_Or, TC_Unary, T_DollarArithmetic, T_DollarBraceCommandExpansion, T_DollarBraced, T_DollarBracket, T_DollarDoubleQuoted, T_DollarExpansion, T_DoubleQuoted, T_Extglob, T_FdRedirect, T_ForArithmetic, T_ForIn, T_Function, T_HereDoc, T_HereString, T_IfExpression, T_Include, T_IndexedElement, T_IoDuplicate, T_IoFile, T_NormalWord, T_OrIf, T_Pipeline, T_ProcSub, T_Redirecting, T_Script, T_SelectIn, T_SimpleCommand, T_SourceCommand, T_Subshell, T_UntilExpression, T_WhileExpression #-} instance Eq Token where OuterToken _ a == OuterToken _ b = a == b diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 211993c..b3d673f 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -3294,7 +3294,7 @@ checkReturnAgainstZero params token = next@(TA_Unary _ "!" _):_ -> isOnlyTestInCommand next next@(TC_Group {}):_ -> isOnlyTestInCommand next next@(TA_Sequence _ [_]):_ -> isOnlyTestInCommand next - next@(TA_Parentesis _ _):_ -> isOnlyTestInCommand next + next@(TA_Parenthesis _ _):_ -> isOnlyTestInCommand next _ -> False -- TODO: Do better $? tracking and filter on whether @@ -4990,14 +4990,14 @@ checkUnnecessaryParens params t = T_ForArithmetic _ x y z _ -> mapM_ (checkLeading "for (((x); (y); (z))) is the same as for ((x; y; z))") [x,y,z] T_Assignment _ _ _ [t] _ -> checkLeading "a[(x)] is the same as a[x]" t T_Arithmetic _ t -> checkLeading "(( (x) )) is the same as (( x ))" t - TA_Parentesis _ (TA_Sequence _ [ TA_Parentesis id _ ]) -> + TA_Parenthesis _ (TA_Sequence _ [ TA_Parenthesis id _ ]) -> styleWithFix id 2322 "In arithmetic contexts, ((x)) is the same as (x). Prefer only one layer of parentheses." $ fix id _ -> return () where checkLeading str t = case t of - TA_Sequence _ [TA_Parentesis id _ ] -> styleWithFix id 2323 (str ++ ". Prefer not wrapping in additional parentheses.") $ fix id + TA_Sequence _ [TA_Parenthesis id _ ] -> styleWithFix id 2323 (str ++ ". Prefer not wrapping in additional parentheses.") $ fix id _ -> return () fix id = diff --git a/src/ShellCheck/CFG.hs b/src/ShellCheck/CFG.hs index dc56b58..81689a1 100644 --- a/src/ShellCheck/CFG.hs +++ b/src/ShellCheck/CFG.hs @@ -490,7 +490,7 @@ build t = do TA_Binary _ _ a b -> sequentially [a,b] TA_Expansion _ list -> sequentially list TA_Sequence _ list -> sequentially list - TA_Parentesis _ t -> build t + TA_Parenthesis _ t -> build t TA_Trinary _ cond a b -> do condition <- build cond diff --git a/src/ShellCheck/Checks/Custom.hs b/src/ShellCheck/Checks/Custom.hs index 76ac83c..17e9c9e 100644 --- a/src/ShellCheck/Checks/Custom.hs +++ b/src/ShellCheck/Checks/Custom.hs @@ -1,7 +1,7 @@ {- This empty file is provided for ease of patching in site specific checks. However, there are no guarantees regarding compatibility between versions. --} +-} {-# LANGUAGE TemplateHaskell #-} module ShellCheck.Checks.Custom (checker, ShellCheck.Checks.Custom.runTests) where diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 9cc5e02..dfe3131 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -827,7 +827,7 @@ readArithmeticContents = char ')' id <- endSpan start spacing - return $ TA_Parentesis id s + return $ TA_Parenthesis id s readArithTerm = readGroup <|> readVariable <|> readExpansion From c7611dfcc6ccb320b530a4e9179e6facee96a422 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Mon, 19 Aug 2024 18:37:29 -0700 Subject: [PATCH 420/459] Use dynamic artifact name to work around issue with v4 uploader --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8f9d7d0..3886655 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -65,7 +65,7 @@ jobs: - name: Upload artifact uses: actions/upload-artifact@v4 with: - name: bin + name: ${{matrix.build}}.bin path: bin/ package_binary: From 68e6f02267defb1e2e6398b0f477b4b85e930c22 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 31 Aug 2024 18:00:49 -0700 Subject: [PATCH 421/459] Expand list of recognized unicode spaces (and rewrite for performance) --- src/ShellCheck/Parser.hs | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index dfe3131..3de3537 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -141,15 +141,9 @@ carriageReturn = do parseProblemAt pos ErrorC 1017 "Literal carriage return. Run script through tr -d '\\r' ." return '\r' -almostSpace = - choice [ - check '\xA0' "unicode non-breaking space", - check '\x200B' "unicode zerowidth space" - ] - where - check c name = do - parseNote ErrorC 1018 $ "This is a " ++ name ++ ". Delete and retype it." - char c +almostSpace = do + parseNote ErrorC 1018 $ "This is a unicode space. Delete and retype it." + oneOf "\xA0\x2002\x2003\x2004\x2005\x2006\x2007\x2008\x2009\x200B\x202F" return ' ' --------- Message/position annotation on top of user state From 1487e57a46a48f926f0cd698756cfe41d9635c15 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 31 Aug 2024 18:27:18 -0700 Subject: [PATCH 422/459] Suppress unused warnings about stderr and stderr_lines from bats tests, fixing tests. --- src/ShellCheck/Data.hs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/ShellCheck/Data.hs b/src/ShellCheck/Data.hs index 917142e..3000a99 100644 --- a/src/ShellCheck/Data.hs +++ b/src/ShellCheck/Data.hs @@ -62,6 +62,9 @@ internalVariables = [ , "FLAGS_ARGC", "FLAGS_ARGV", "FLAGS_ERROR", "FLAGS_FALSE", "FLAGS_HELP", "FLAGS_PARENT", "FLAGS_RESERVED", "FLAGS_TRUE", "FLAGS_VERSION", "flags_error", "flags_return" + + -- Bats + ,"stderr", "stderr_lines" ] specialIntegerVariables = [ From 88e441453ba3bbb2aa0449dc178b8f82aaee5b4c Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 31 Aug 2024 18:31:47 -0700 Subject: [PATCH 423/459] Make SC2002 optional (useless-use-of-cat) --- src/ShellCheck/Analytics.hs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 869b683..ce11884 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -103,8 +103,7 @@ nodeChecksToTreeCheck checkList = nodeChecks :: [Parameters -> Token -> Writer [TokenComment] ()] nodeChecks = [ - checkUuoc - ,checkPipePitfalls + checkPipePitfalls ,checkForInQuoted ,checkForInLs ,checkShorthandIf @@ -273,6 +272,13 @@ optionalTreeChecks = [ cdPositive = "rm -r \"$(get_chroot_dir)/home\"", cdNegative = "set -e; dir=\"$(get_chroot_dir)\"; rm -r \"$dir/home\"" }, checkExtraMaskedReturns) + + ,(newCheckDescription { + cdName = "useless-use-of-cat", + cdDescription = "Check for Useless Use Of Cat (UUOC)", + cdPositive = "cat foo | grep bar", + cdNegative = "grep bar foo" + }, nodeChecksToTreeCheck [checkUuoc]) ] optionalCheckMap :: Map.Map String (Parameters -> Token -> [TokenComment]) From 8a1b24c7afcd0534e15eb7db8e386d0d550c6015 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 1 Sep 2024 13:21:44 -0700 Subject: [PATCH 424/459] Fix paths for CI binary packaging after upgrade --- .github/workflows/build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3886655..b0d1085 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -43,7 +43,7 @@ jobs: path: source/ build_source: - name: Build Source Code + name: Build needs: package_source strategy: matrix: @@ -80,13 +80,13 @@ jobs: uses: actions/download-artifact@v4 - name: Work around GitHub permissions bug - run: chmod +x bin/*/shellcheck* + run: chmod +x *.bin/*/shellcheck* - name: Package binaries run: | export TAGS="$(cat source/tags)" mkdir -p deploy - cp -r bin/* deploy + cp -r *.bin/* deploy cd deploy ../.prepare_deploy rm -rf */ README* LICENSE* From ca65071d778d140bc6bad64e699265dde25821e9 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 1 Sep 2024 14:06:26 -0700 Subject: [PATCH 425/459] Run unit tests in GitHub actions --- .github/workflows/build.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b0d1085..83269c9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -42,6 +42,29 @@ jobs: name: source path: source/ + run_tests: + name: Run tests + needs: package_source + runs-on: ubuntu-latest + steps: + - name: Download artifacts + uses: actions/download-artifact@v4 + + - name: Install dependencies + run: | + sudo apt-get update && sudo apt-get install ghc cabal-install + cabal update + + - name: Unpack source + run: | + cd source + tar xvf source.tar.gz --strip-components=1 + + - name: Build and run tests + run: | + cd source + cabal test + build_source: name: Build needs: package_source From 79e43c4550aaf50ebcced1c2b2852f1bd2533f6c Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 7 Sep 2024 17:14:52 -0700 Subject: [PATCH 426/459] Allow parsing arbitrary coproc names (fixes #3048) --- src/ShellCheck/AST.hs | 2 +- src/ShellCheck/AnalyzerLib.hs | 8 ++++++-- src/ShellCheck/CFG.hs | 14 +++++++++++--- src/ShellCheck/Parser.hs | 35 ++++++++++++++++++++++++++++------- 4 files changed, 46 insertions(+), 13 deletions(-) diff --git a/src/ShellCheck/AST.hs b/src/ShellCheck/AST.hs index 979a9b0..bafe035 100644 --- a/src/ShellCheck/AST.hs +++ b/src/ShellCheck/AST.hs @@ -138,7 +138,7 @@ data InnerToken t = | Inner_T_WhileExpression [t] [t] | Inner_T_Annotation [Annotation] t | Inner_T_Pipe String - | Inner_T_CoProc (Maybe String) t + | Inner_T_CoProc (Maybe Token) t | Inner_T_CoProcBody t | Inner_T_Include t | Inner_T_SourceCommand t t diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 3b1faa9..531ce8b 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -559,8 +559,12 @@ getModifiedVariables t = T_FdRedirect _ ('{':var) op -> -- {foo}>&2 modifies foo [(t, t, takeWhile (/= '}') var, DataString SourceInteger) | not $ isClosingFileOp op] - T_CoProc _ name _ -> - [(t, t, fromMaybe "COPROC" name, DataArray SourceInteger)] + T_CoProc _ Nothing _ -> + [(t, t, "COPROC", DataArray SourceInteger)] + + T_CoProc _ (Just token) _ -> do + name <- maybeToList $ getLiteralString token + [(t, t, name, DataArray SourceInteger)] --Points to 'for' rather than variable T_ForIn id str [] _ -> [(t, t, str, DataString SourceExternal)] diff --git a/src/ShellCheck/CFG.hs b/src/ShellCheck/CFG.hs index 81689a1..57aaf4b 100644 --- a/src/ShellCheck/CFG.hs +++ b/src/ShellCheck/CFG.hs @@ -668,10 +668,18 @@ build t = do status <- newNodeRange $ CFSetExitCode id linkRange cond status - T_CoProc id maybeName t -> do - let name = fromMaybe "COPROC" maybeName + T_CoProc id maybeNameToken t -> do + -- If unspecified, "COPROC". If not a constant string, Nothing. + let maybeName = case maybeNameToken of + Just x -> getLiteralString x + Nothing -> Just "COPROC" + + let parentNode = case maybeName of + Just str -> applySingle $ IdTagged id $ CFWriteVariable str CFValueArray + Nothing -> CFStructuralNode + start <- newStructuralNode - parent <- newNodeRange $ applySingle $ IdTagged id $ CFWriteVariable name CFValueArray + parent <- newNodeRange parentNode child <- subshell id "coproc" $ build t end <- newNodeRange $ CFSetExitCode id diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 3de3537..66d62ff 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -2795,17 +2795,29 @@ readFunctionDefinition = called "function" $ do prop_readCoProc1 = isOk readCoProc "coproc foo { echo bar; }" prop_readCoProc2 = isOk readCoProc "coproc { echo bar; }" prop_readCoProc3 = isOk readCoProc "coproc echo bar" +prop_readCoProc4 = isOk readCoProc "coproc a=b echo bar" +prop_readCoProc5 = isOk readCoProc "coproc 'foo' { echo bar; }" +prop_readCoProc6 = isOk readCoProc "coproc \"foo$$\" { echo bar; }" +prop_readCoProc7 = isOk readCoProc "coproc 'foo' ( echo bar )" +prop_readCoProc8 = isOk readCoProc "coproc \"foo$$\" while true; do true; done" readCoProc = called "coproc" $ do start <- startSpan try $ do string "coproc" - whitespace + spacing1 choice [ try $ readCompoundCoProc start, readSimpleCoProc start ] where readCompoundCoProc start = do - var <- optionMaybe $ - readVariableName `thenSkip` whitespace - body <- readBody readCompoundCommand + notFollowedBy2 readAssignmentWord + (var, body) <- choice [ + try $ do + body <- readBody readCompoundCommand + return (Nothing, body), + try $ do + var <- readNormalWord `thenSkip` spacing + body <- readBody readCompoundCommand + return (Just var, body) + ] id <- endSpan start return $ T_CoProc id var body readSimpleCoProc start = do @@ -3436,13 +3448,22 @@ isOk p s = parsesCleanly p s == Just True -- The string parses with no wa isWarning p s = parsesCleanly p s == Just False -- The string parses with warnings isNotOk p s = parsesCleanly p s == Nothing -- The string does not parse -parsesCleanly parser string = runIdentity $ do +-- If the parser matches the string, return Right [ParseNotes+ParseProblems] +-- If it does not match the string, return Left [ParseProblems] +getParseOutput parser string = runIdentity $ do (res, sys) <- runParser testEnvironment (parser >> eof >> getState) "-" string case (res, sys) of (Right userState, systemState) -> - return $ Just . null $ parseNotes userState ++ parseProblems systemState - (Left _, _) -> return Nothing + return $ Right $ parseNotes userState ++ parseProblems systemState + (Left _, systemState) -> return $ Left $ parseProblems systemState + +-- If the parser matches the string, return Just whether it was clean (without emitting suggestions) +-- Otherwise, Nothing +parsesCleanly parser string = + case getParseOutput parser string of + Right list -> Just $ null list + Left _ -> Nothing parseWithNotes parser = do item <- parser From 5c2be767abff85fd10325a2d8a61a372567e63b8 Mon Sep 17 00:00:00 2001 From: Tony <3987237+random1223@users.noreply.github.com> Date: Mon, 9 Sep 2024 18:56:18 -0700 Subject: [PATCH 427/459] Update README.md Add Codety Scanner into the static analysis solution list. Here are the examples of the result: * Codety's pull request code review example: https://github.com/codetyio/codety-scanner/pull/66#issuecomment-2339438925 * Codety's GitHub code scan result example : https://github.com/codetyio/codety-scanner/runs/29907371258 Codety Scanner is open source: https://github.com/codetyio/codety-scanner --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c200a3e..9b776cf 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,7 @@ Services and platforms that have ShellCheck pre-installed and ready to use: * [Codacy](https://www.codacy.com/) * [Code Climate](https://codeclimate.com/) * [Code Factor](https://www.codefactor.io/) +* [Codety](https://www.codety.io/) via the [Codety Scanner](https://github.com/codetyio/codety-scanner) * [CircleCI](https://circleci.com) via the [ShellCheck Orb](https://circleci.com/orbs/registry/orb/circleci/shellcheck) * [Github](https://github.com/features/actions) (only Linux) * [Trunk Check](https://trunk.io/products/check) (universal linter; [allows you to explicitly version your shellcheck install](https://github.com/trunk-io/plugins/blob/bcbb361dcdbe4619af51ea7db474d7fb87540d20/.trunk/trunk.yaml#L32)) via the [shellcheck plugin](https://github.com/trunk-io/plugins/blob/main/linters/shellcheck/plugin.yaml) From 5e3e98bcb0ca594185e9e675dac929aa053dd223 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 27 Oct 2024 15:43:30 -0700 Subject: [PATCH 428/459] Use CFG to determine use-before-define for SC2218 (fixes #3070) --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 48 ++++++++++++++++++------------------- 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 43db00c..982036d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ ### Changed - SC2015 about `A && B || C` no longer triggers when B is a test command. ### Fixed +- SC2218 about function use-before-define is now more accurate. - SC2317 about unreachable commands is now less spammy for nested ones. - SC2292, optional suggestion for [[ ]], now triggers for Busybox. diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index ce11884..7f7a572 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -3765,32 +3765,32 @@ prop_checkUseBeforeDefinition1 = verifyTree checkUseBeforeDefinition "f; f() { t prop_checkUseBeforeDefinition2 = verifyNotTree checkUseBeforeDefinition "f() { true; }; f" prop_checkUseBeforeDefinition3 = verifyNotTree checkUseBeforeDefinition "if ! mycmd --version; then mycmd() { true; }; fi" prop_checkUseBeforeDefinition4 = verifyNotTree checkUseBeforeDefinition "mycmd || mycmd() { f; }" -checkUseBeforeDefinition _ t = - execWriter $ evalStateT (mapM_ examine $ revCommands) Map.empty +prop_checkUseBeforeDefinition5 = verifyTree checkUseBeforeDefinition "false || mycmd; mycmd() { f; }" +prop_checkUseBeforeDefinition6 = verifyNotTree checkUseBeforeDefinition "f() { one; }; f; f() { two; }; f" +checkUseBeforeDefinition :: Parameters -> Token -> [TokenComment] +checkUseBeforeDefinition params t = fromMaybe [] $ do + cfga <- cfgAnalysis params + let funcs = execState (doAnalysis findFunction t) Map.empty + -- Green cut: no point enumerating commands if there are no functions. + guard . not $ Map.null funcs + return $ execWriter $ doAnalysis (findInvocation cfga funcs) t where - examine t = case t of - T_Pipeline _ _ [T_Redirecting _ _ (T_Function _ _ _ name _)] -> - modify $ Map.insert name t - T_Annotation _ _ w -> examine w - T_Pipeline _ _ cmds -> do - m <- get - unless (Map.null m) $ - mapM_ (checkUsage m) $ concatMap recursiveSequences cmds - _ -> return () + findFunction t = + case t of + T_Function id _ _ name _ -> modify (Map.insertWith (++) name [id]) + _ -> return () - checkUsage map cmd = sequence_ $ do - name <- getCommandName cmd - def <- Map.lookup name map - return $ - err (getId cmd) 2218 - "This function is only defined later. Move the definition up." - - revCommands = reverse $ concat $ getCommandSequences t - recursiveSequences x = - let list = concat $ getCommandSequences x in - if null list - then [x] - else concatMap recursiveSequences list + findInvocation cfga funcs t = + case t of + T_SimpleCommand id _ (cmd:_) -> sequence_ $ do + name <- getLiteralString cmd + invocations <- Map.lookup name funcs + -- Is the function definitely being defined later? + guard $ any (\c -> CF.doesPostDominate cfga c id) invocations + -- Was one already defined, so it's actually a re-definition? + guard . not $ any (\c -> CF.doesPostDominate cfga id c) invocations + return $ err id 2218 "This function is only defined later. Move the definition up." + _ -> return () prop_checkForLoopGlobVariables1 = verify checkForLoopGlobVariables "for i in $var/*.txt; do true; done" prop_checkForLoopGlobVariables2 = verifyNot checkForLoopGlobVariables "for i in \"$var\"/*.txt; do true; done" From f2932ebcdc51527ca439737465e35b0b039a51b7 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 27 Oct 2024 16:02:56 -0700 Subject: [PATCH 429/459] Remember to add changelog to release messages (fixes #3051) --- test/check_release | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/check_release b/test/check_release index 665b265..f3ea9df 100755 --- a/test/check_release +++ b/test/check_release @@ -50,6 +50,11 @@ then fail "Expected git log message to be 'Stable version ...'" fi +if [[ $(git log -1 --pretty=%B) != *"CHANGELOG"* ]] +then + fail "Expected git log message to contain CHANGELOG" +fi + i=1 j=1 cat << EOF From 097018754b313a102834ec389079ee04673f68fa Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 27 Oct 2024 18:10:00 -0700 Subject: [PATCH 430/459] Mention that SC2002 (UUOC) is now no longer enabled by default. --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 982036d..612068d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ - SC2330: Warn about unsupported glob matches with [[ .. ]] in BusyBox. - Precompiled binaries for Linux riscv64 (linux.riscv64) ### Changed +- SC2002 about Useless Use Of Cat is now disabled by default. It can be + re-enabled with `--enable=useless-use-of-cat` or equivalent directive. - SC2015 about `A && B || C` no longer triggers when B is a test command. ### Fixed - SC2218 about function use-before-define is now more accurate. From 792466bc22ea9528313b5224621610551216e4a6 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 3 Nov 2024 13:56:51 -0800 Subject: [PATCH 431/459] Update Diff dependency (fixes #3075) --- ShellCheck.cabal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ShellCheck.cabal b/ShellCheck.cabal index fc52b12..0f604a9 100644 --- a/ShellCheck.cabal +++ b/ShellCheck.cabal @@ -53,7 +53,7 @@ library bytestring >= 0.10.6 && < 0.13, containers >= 0.5.6 && < 0.8, deepseq >= 1.4.1 && < 1.6, - Diff >= 0.4.0 && < 0.6, + Diff >= 0.4.0 && < 1.1, fgl (>= 5.7.0 && < 5.8.1.0) || (>= 5.8.1.1 && < 5.9), filepath >= 1.4.0 && < 1.5, mtl >= 2.2.2 && < 2.4, From 0ee46a0f33ebafde128e2c93dd45f2757de4d4ec Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 3 Nov 2024 14:19:08 -0800 Subject: [PATCH 432/459] Update filepath dependency --- ShellCheck.cabal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ShellCheck.cabal b/ShellCheck.cabal index 0f604a9..f31ead3 100644 --- a/ShellCheck.cabal +++ b/ShellCheck.cabal @@ -55,7 +55,7 @@ library deepseq >= 1.4.1 && < 1.6, Diff >= 0.4.0 && < 1.1, fgl (>= 5.7.0 && < 5.8.1.0) || (>= 5.8.1.1 && < 5.9), - filepath >= 1.4.0 && < 1.5, + filepath >= 1.4.0 && < 1.6, mtl >= 2.2.2 && < 2.4, parsec >= 3.1.14 && < 3.2, QuickCheck >= 2.14.2 && < 2.15, From 47bff1d5fdc478a3bfb32ffb532d33bab0e64b2c Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 3 Nov 2024 16:54:45 -0800 Subject: [PATCH 433/459] Add 24.04 to distrotest LTS --- test/distrotest | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/distrotest b/test/distrotest index ef467b8..c52a5c9 100755 --- a/test/distrotest +++ b/test/distrotest @@ -74,11 +74,12 @@ fedora:latest dnf install -y cabal-install ghc-template-haskell-devel fi archlinux:latest pacman -S -y --noconfirm cabal-install ghc-static base-devel # Ubuntu LTS +ubuntu:24.04 apt-get update && apt-get install -y cabal-install ubuntu:22.04 apt-get update && apt-get install -y cabal-install ubuntu:20.04 apt-get update && apt-get install -y cabal-install # Stack on Ubuntu LTS -ubuntu:22.04 set -e; apt-get update && apt-get install -y curl && curl -sSL https://get.haskellstack.org/ | sh -s - -f && cd /mnt && exec test/stacktest +ubuntu:24.04 set -e; apt-get update && apt-get install -y curl && curl -sSL https://get.haskellstack.org/ | sh -s - -f && cd /mnt && exec test/stacktest EOF exit "$final" From 944d87915a191af442b3895ef5af89ffb65789d8 Mon Sep 17 00:00:00 2001 From: Evan Silberman Date: Mon, 11 Nov 2024 11:24:21 -0800 Subject: [PATCH 434/459] Recognize "oksh" executable name as ksh A portable version of OpenBSD's ksh is distributed with the executable name oksh [1]. It's a descendant of pdksh and can be shellchecked as ksh. [1]: https://github.com/ibara/oksh --- src/ShellCheck/Data.hs | 1 + src/ShellCheck/Parser.hs | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Data.hs b/src/ShellCheck/Data.hs index 3000a99..a145684 100644 --- a/src/ShellCheck/Data.hs +++ b/src/ShellCheck/Data.hs @@ -167,6 +167,7 @@ shellForExecutable name = "ksh" -> return Ksh "ksh88" -> return Ksh "ksh93" -> return Ksh + "oksh" -> return Ksh _ -> Nothing flagsForRead = "sreu:n:N:i:p:a:t:" diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 66d62ff..3986dab 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -3387,7 +3387,8 @@ readScriptFile sourced = do "busybox sh", "bash", "bats", - "ksh" + "ksh", + "oksh" ] badShells = [ "awk", From 7f3f014d49d4bc6b979ed5508b1c57874e795ee8 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Thu, 28 Nov 2024 11:51:22 -0800 Subject: [PATCH 435/459] Allow latest QuickCheck --- ShellCheck.cabal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ShellCheck.cabal b/ShellCheck.cabal index f31ead3..68c32d9 100644 --- a/ShellCheck.cabal +++ b/ShellCheck.cabal @@ -58,7 +58,7 @@ library filepath >= 1.4.0 && < 1.6, mtl >= 2.2.2 && < 2.4, parsec >= 3.1.14 && < 3.2, - QuickCheck >= 2.14.2 && < 2.15, + QuickCheck >= 2.14.2 && < 2.16, regex-tdfa >= 1.2.0 && < 1.4, transformers >= 0.4.2 && < 0.7, From 3c75d82db571aa3fd337e04077daa3c01e0c878e Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Fri, 29 Nov 2024 12:58:56 -0800 Subject: [PATCH 436/459] Fix stacktest complaining about permissions on /mnt --- test/distrotest | 6 +++--- test/stacktest | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/distrotest b/test/distrotest index c52a5c9..128ee44 100755 --- a/test/distrotest +++ b/test/distrotest @@ -17,13 +17,13 @@ and is still highly experimental. Make sure you're plugged in and have screen/tmux in place, then re-run with $0 --run to continue. -Also note that dist* will be deleted. +Also note that dist*/ and .stack-work/ will be deleted. EOF exit 0 } -echo "Deleting 'dist' and 'dist-newstyle'..." -rm -rf dist dist-newstyle +echo "Deleting 'dist', 'dist-newstyle', and '.stack-work'..." +rm -rf dist dist-newstyle .stack-work execs=$(find . -name shellcheck) diff --git a/test/stacktest b/test/stacktest index 9eb8d1e..b486c31 100755 --- a/test/stacktest +++ b/test/stacktest @@ -15,7 +15,7 @@ die() { echo "$*" >&2; exit 1; } command -v stack || die "stack is missing" -stack setup || die "Failed to setup with default resolver" +stack setup --allow-different-user || die "Failed to setup with default resolver" stack build --test || die "Failed to build/test with default resolver" # Nice to haves, but not necessary From 195b70db8c697e44b65beeb83abe4c283cd4cda4 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Fri, 13 Dec 2024 23:06:49 -0500 Subject: [PATCH 437/459] Use unless instead of when and not --- src/ShellCheck/Checks/ShellSupport.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 2ec4cf7..f228832 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -330,7 +330,7 @@ checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do | t `isCommand` "echo" && argString `matches` flagRegex = if isBusyboxSh then - when (not (argString `matches` busyboxFlagRegex)) $ + unless (argString `matches` busyboxFlagRegex) $ warnMsg (getId arg) 3036 "echo flags besides -n and -e" else if isDash then From 0ecaf2b5f165497e461a47330ebbe11b05d8eb1a Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Fri, 13 Dec 2024 23:19:36 -0500 Subject: [PATCH 438/459] Use foldr instead of explicit recursion --- src/ShellCheck/Analytics.hs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 7f7a572..3a47ed9 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -5074,10 +5074,9 @@ checkExpansionWithRedirection params t = (T_Pipeline _ _ t@(_:_)) -> checkCmd id (last t) _ -> return () - checkCmd captureId (T_Redirecting _ redirs _) = walk captureId redirs + checkCmd captureId (T_Redirecting _ redirs _) = foldr (walk captureId) (return ()) redirs - walk captureId [] = return () - walk captureId (t:rest) = + walk captureId t acc = case t of T_FdRedirect _ _ (T_IoDuplicate _ _ "1") -> return () T_FdRedirect id "1" (T_IoDuplicate _ _ _) -> return () @@ -5086,7 +5085,7 @@ checkExpansionWithRedirection params t = if getLiteralString file == Just "/dev/null" then emit id captureId False else emit id captureId True - _ -> walk captureId rest + _ -> acc emit redirectId captureId suggestTee = do warn captureId 2327 "This command substitution will be empty because the command's output gets redirected away." From 5adfea21eec84a6bc8bbcaf4e22c02461dd850d9 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Fri, 13 Dec 2024 23:20:48 -0500 Subject: [PATCH 439/459] Use the result of the comparison directly instead of an if/else --- src/ShellCheck/Analytics.hs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 3a47ed9..4757e57 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -5082,9 +5082,7 @@ checkExpansionWithRedirection params t = T_FdRedirect id "1" (T_IoDuplicate _ _ _) -> return () T_FdRedirect id "" (T_IoDuplicate _ op _) | op `elem` [T_GREATAND (Id 0), T_Greater (Id 0)] -> emit id captureId True T_FdRedirect id str (T_IoFile _ op file) | str `elem` ["", "1"] && op `elem` [ T_DGREAT (Id 0), T_Greater (Id 0) ] -> - if getLiteralString file == Just "/dev/null" - then emit id captureId False - else emit id captureId True + emit id captureId $ getLiteralString file /= Just "/dev/null" _ -> acc emit redirectId captureId suggestTee = do From 26b949b9b0b1552639fdf24411096cf00749be18 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Fri, 13 Dec 2024 23:45:32 -0500 Subject: [PATCH 440/459] Use mapM_ instead of isJust and fromJust --- src/ShellCheck/Checks/Commands.hs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index c10016e..c37a67d 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -1431,9 +1431,8 @@ prop_checkBackreferencingDeclaration7 = verify (checkBackreferencingDeclaration checkBackreferencingDeclaration cmd = CommandCheck (Exactly cmd) check where check t = do - cfga <- asks cfgAnalysis - when (isJust cfga) $ - foldM_ (perArg $ fromJust cfga) M.empty $ arguments t + maybeCfga <- asks cfgAnalysis + mapM_ (\cfga -> foldM_ (perArg cfga) M.empty $ arguments t) maybeCfga perArg cfga leftArgs t = case t of From 7deb7e853b4bc5d2ad90f021de180055e87611b5 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Fri, 13 Dec 2024 23:47:55 -0500 Subject: [PATCH 441/459] Use mapM_ instead of sequence_ and <$> --- src/ShellCheck/Checks/ControlFlow.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Checks/ControlFlow.hs b/src/ShellCheck/Checks/ControlFlow.hs index d23fa15..9f63141 100644 --- a/src/ShellCheck/Checks/ControlFlow.hs +++ b/src/ShellCheck/Checks/ControlFlow.hs @@ -78,7 +78,7 @@ controlFlowEffectChecks = [ runNodeChecks :: [ControlFlowNodeCheck] -> ControlFlowCheck runNodeChecks perNode = do cfg <- asks cfgAnalysis - sequence_ $ runOnAll <$> cfg + mapM_ runOnAll cfg where getData datas n@(node, label) = do (pre, post) <- M.lookup node datas From d3001f337aa3f7653a621b302261f4eac01890d0 Mon Sep 17 00:00:00 2001 From: "Joseph C. Sible" Date: Fri, 13 Dec 2024 23:57:50 -0500 Subject: [PATCH 442/459] Simplify getParseOutput --- src/ShellCheck/Parser.hs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 66d62ff..9628b2e 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -3451,12 +3451,12 @@ isNotOk p s = parsesCleanly p s == Nothing -- The string does not parse -- If the parser matches the string, return Right [ParseNotes+ParseProblems] -- If it does not match the string, return Left [ParseProblems] getParseOutput parser string = runIdentity $ do - (res, sys) <- runParser testEnvironment - (parser >> eof >> getState) "-" string - case (res, sys) of - (Right userState, systemState) -> - return $ Right $ parseNotes userState ++ parseProblems systemState - (Left _, systemState) -> return $ Left $ parseProblems systemState + (res, systemState) <- runParser testEnvironment + (parser >> eof >> getState) "-" string + return $ case res of + Right userState -> + Right $ parseNotes userState ++ parseProblems systemState + Left _ -> Left $ parseProblems systemState -- If the parser matches the string, return Just whether it was clean (without emitting suggestions) -- Otherwise, Nothing From fe315a25c42219b8a7b0b16ffa69351421e2e997 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lawrence=20Vel=C3=A1zquez?= Date: Sat, 28 Dec 2024 03:05:07 -0500 Subject: [PATCH 443/459] Recognize internal variables new in bash 5.3 From the bug-bash@gnu.org announcement "Bash-5.3-beta available": q. GLOBSORT: new variable to specify how to sort the results of pathname expansion (name, size, blocks, mtime, atime, ctime, none) in ascending or descending order. w. BASH_MONOSECONDS: new dynamic variable that returns the value of the system's monotonic clock, if one is available. x. BASH_TRAPSIG: new variable, set to the numeric signal number of the trap being executed while it's running. https://lists.gnu.org/archive/html/bug-bash/2024-12/msg00120.html --- src/ShellCheck/Data.hs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Data.hs b/src/ShellCheck/Data.hs index 3000a99..3876507 100644 --- a/src/ShellCheck/Data.hs +++ b/src/ShellCheck/Data.hs @@ -49,6 +49,7 @@ internalVariables = [ "LINES", "MAIL", "MAILCHECK", "MAILPATH", "OPTERR", "PATH", "POSIXLY_CORRECT", "PROMPT_COMMAND", "PROMPT_DIRTRIM", "PS0", "PS1", "PS2", "PS3", "PS4", "SHELL", "TIMEFORMAT", "TMOUT", "TMPDIR", + "BASH_MONOSECONDS", "BASH_TRAPSIG", "GLOBSORT", "auto_resume", "histchars", -- Other @@ -78,7 +79,7 @@ variablesWithoutSpaces = specialVariablesWithoutSpaces ++ [ "EPOCHREALTIME", "EPOCHSECONDS", "LINENO", "OPTIND", "PPID", "RANDOM", "READLINE_ARGUMENT", "READLINE_MARK", "READLINE_POINT", "SECONDS", "SHELLOPTS", "SHLVL", "SRANDOM", "UID", "COLUMNS", "HISTFILESIZE", - "HISTSIZE", "LINES" + "HISTSIZE", "LINES", "BASH_MONOSECONDS", "BASH_TRAPSIG" -- shflags , "FLAGS_ERROR", "FLAGS_FALSE", "FLAGS_TRUE" From cbf0b33463601fbd9827db31e08db424e3381074 Mon Sep 17 00:00:00 2001 From: Adrian Fluturel Date: Tue, 7 Jan 2025 03:24:29 +0100 Subject: [PATCH 444/459] Skip SC2015 when the last command is true --- src/ShellCheck/Analytics.hs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 4757e57..1329c86 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -881,13 +881,15 @@ prop_checkShorthandIf6 = verifyNot checkShorthandIf "if foo && bar || baz; then prop_checkShorthandIf7 = verifyNot checkShorthandIf "while foo && bar || baz; do true; done" prop_checkShorthandIf8 = verify checkShorthandIf "if true; then foo && bar || baz; fi" prop_checkShorthandIf9 = verifyNot checkShorthandIf "foo && [ -x /file ] || bar" +prop_checkShorthandIf10 = verifyNot checkShorthandIf "foo && bar || true" +prop_checkShorthandIf11 = verify checkShorthandIf "foo && bar || false" checkShorthandIf params x@(T_OrIf _ (T_AndIf id _ b) (T_Pipeline _ _ t)) | not (isOk t || inCondition) && not (isTestCommand b) = info id 2015 "Note that A && B || C is not if-then-else. C may run when A is true." where isOk [t] = isAssignment t || fromMaybe False (do name <- getCommandBasename t - return $ name `elem` ["echo", "exit", "return", "printf"]) + return $ name `elem` ["echo", "exit", "return", "printf", "true"]) isOk _ = False inCondition = isCondition $ getPath (parentMap params) x checkShorthandIf _ _ = return () From 3a9ddae06b7e2293ebf61eaf8c71b01bbb769614 Mon Sep 17 00:00:00 2001 From: Eisuke Kawashima Date: Mon, 24 Mar 2025 05:49:06 +0900 Subject: [PATCH 445/459] fix(SC3013)!: remove SC3013 since the operators are specified by POSIX.1-2024 https://pubs.opengroup.org/onlinepubs/9799919799/utilities/test.html fix #3167 --- CHANGELOG.md | 2 ++ src/ShellCheck/Checks/ShellSupport.hs | 8 +------- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 612068d..1a24b53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ - SC2317 about unreachable commands is now less spammy for nested ones. - SC2292, optional suggestion for [[ ]], now triggers for Busybox. +### Removed +- SC3013: removed since the operators `-op/-nt/-ef` are specified in POSIX.1-2024 ## v0.10.0 - 2024-03-07 ### Added diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index f228832..1789b6f 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -86,7 +86,7 @@ checkForDecimals = ForShell [Sh, Dash, BusyboxSh, Bash] f prop_checkBashisms = verify checkBashisms "while read a; do :; done < <(a)" -prop_checkBashisms2 = verify checkBashisms "[ foo -nt bar ]" +prop_checkBashisms2 = verifyNot checkBashisms "[ foo -nt bar ]" prop_checkBashisms3 = verify checkBashisms "echo $((i++))" prop_checkBashisms4 = verify checkBashisms "rm !(*.hs)" prop_checkBashisms5 = verify checkBashisms "source file" @@ -252,12 +252,6 @@ checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do bashism (T_SimpleCommand id _ [asStr -> Just "test", lhs, asStr -> Just op, rhs]) | op `elem` [ "<", ">", "\\<", "\\>", "<=", ">=", "\\<=", "\\>="] = unless isDash $ warnMsg id 3012 $ "lexicographical " ++ op ++ " is" - bashism (TC_Binary id SingleBracket op _ _) - | op `elem` [ "-ot", "-nt", "-ef" ] = - unless isDash $ warnMsg id 3013 $ op ++ " is" - bashism (T_SimpleCommand id _ [asStr -> Just "test", lhs, asStr -> Just op, rhs]) - | op `elem` [ "-ot", "-nt", "-ef" ] = - unless isDash $ warnMsg id 3013 $ op ++ " is" bashism (TC_Binary id SingleBracket "==" _ _) = unless isBusyboxSh $ warnMsg id 3014 "== in place of = is" bashism (T_SimpleCommand id _ [asStr -> Just "test", lhs, asStr -> Just "==", rhs]) = From bc60607f9e5be0ea7528589410845aef2d6f10a3 Mon Sep 17 00:00:00 2001 From: Eisuke Kawashima Date: Mon, 24 Mar 2025 06:04:19 +0900 Subject: [PATCH 446/459] fix(SC3012)!: do not warn about `\<` and `\>` in test/[] as specified in POSIX.1-2024 https://pubs.opengroup.org/onlinepubs/9799919799/utilities/test.html fix #3168 --- CHANGELOG.md | 1 + src/ShellCheck/Checks/ShellSupport.hs | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 612068d..82630bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - SC2002 about Useless Use Of Cat is now disabled by default. It can be re-enabled with `--enable=useless-use-of-cat` or equivalent directive. - SC2015 about `A && B || C` no longer triggers when B is a test command. +- SC3012: Do not warn about `\<` and `\>` in test/[] as specified in POSIX.1-2024 ### Fixed - SC2218 about function use-before-define is now more accurate. - SC2317 about unreachable commands is now less spammy for nested ones. diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index f228832..2a34931 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -247,10 +247,10 @@ checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do unless isBusyboxSh $ warnMsg id 3010 "[[ ]] is" bashism (T_HereString id _) = warnMsg id 3011 "here-strings are" bashism (TC_Binary id SingleBracket op _ _) - | op `elem` [ "<", ">", "\\<", "\\>", "<=", ">=", "\\<=", "\\>="] = + | op `elem` [ "<", ">", "<=", ">=", "\\<=", "\\>="] = unless isDash $ warnMsg id 3012 $ "lexicographical " ++ op ++ " is" bashism (T_SimpleCommand id _ [asStr -> Just "test", lhs, asStr -> Just op, rhs]) - | op `elem` [ "<", ">", "\\<", "\\>", "<=", ">=", "\\<=", "\\>="] = + | op `elem` [ "<", ">", "<=", ">=", "\\<=", "\\>="] = unless isDash $ warnMsg id 3012 $ "lexicographical " ++ op ++ " is" bashism (TC_Binary id SingleBracket op _ _) | op `elem` [ "-ot", "-nt", "-ef" ] = From 4f628cbe2a617e76fe3686a79c3d1eaf443acc08 Mon Sep 17 00:00:00 2001 From: Eisuke Kawashima Date: Fri, 4 Apr 2025 17:31:07 +0900 Subject: [PATCH 447/459] feat: check tautologically-false conditionals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - fix #3179 — negation of SC2055, `[ x = y -a x = z]` - fix #3181 — negation of SC2056, `(( x == y && x == z ))` - fix #3180 — negation of SC2252, `[ x = y ] && [ x = z ]` --- src/ShellCheck/Analytics.hs | 50 +++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 4757e57..431e56b 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -123,6 +123,7 @@ nodeChecks = [ ,checkCaseAgainstGlob ,checkCommarrays ,checkOrNeq + ,checkAndEq ,checkEchoWc ,checkConstantIfs ,checkPipedAssignment @@ -1631,6 +1632,55 @@ checkOrNeq _ (T_OrIf id lhs rhs) = sequence_ $ do checkOrNeq _ _ = return () +prop_checkAndEq1 = verify checkAndEq "if [[ $lol -eq cow && $lol -eq foo ]]; then echo foo; fi" +prop_checkAndEq2 = verify checkAndEq "(( a==lol && a==foo ))" +prop_checkAndEq3 = verify checkAndEq "[ \"$a\" = lol && \"$a\" = foo ]" +prop_checkAndEq4 = verifyNot checkAndEq "[ a = $cow && b = $foo ]" +prop_checkAndEq5 = verifyNot checkAndEq "[[ $a = /home && $a = */public_html/* ]]" +prop_checkAndEq6 = verify checkAndEq "[ $a = a ] && [ $a = b ]" +prop_checkAndEq7 = verify checkAndEq "[ $a = a ] && [ $a = b ] || true" +prop_checkAndEq8 = verifyNot checkAndEq "[[ $a == x && $a == x ]]" +prop_checkAndEq9 = verifyNot checkAndEq "[ 0 -eq $FOO ] && [ 0 -eq $BAR ]" + +-- For test-level "and": [ x = y -a x = z ] +checkAndEq _ (TC_And id typ op (TC_Binary _ _ op1 lhs1 rhs1 ) (TC_Binary _ _ op2 lhs2 rhs2)) + | (op1 == op2 && (op1 == "-eq" || op1 == "=" || op1 == "==")) && lhs1 == lhs2 && rhs1 /= rhs2 && not (any isGlob [rhs1,rhs2]) = + warn id 2055 $ "You probably wanted " ++ (if typ == SingleBracket then "-o" else "||") ++ " here, otherwise it's always false." + +-- For arithmetic context "and" +checkAndEq _ (TA_Binary id "&&" (TA_Binary _ "==" word1 _) (TA_Binary _ "==" word2 _)) + | word1 == word2 = + warn id 2056 "You probably wanted || here, otherwise it's always false." + +-- For command level "and": [ x = y ] && [ x = z ] +checkAndEq _ (T_AndIf id lhs rhs) = sequence_ $ do + (lhs1, op1, rhs1) <- getExpr lhs + (lhs2, op2, rhs2) <- getExpr rhs + guard $ op1 == op2 && op1 `elem` ["-eq", "=", "=="] + guard $ lhs1 == lhs2 && rhs1 /= rhs2 + guard . not $ any isGlob [rhs1, rhs2] + return $ warn id 2252 "You probably wanted || here, otherwise it's always false." + where + getExpr x = + case x of + T_AndIf _ lhs _ -> getExpr lhs -- Fetches x and y in `T_AndIf x (T_AndIf y z)` + T_Pipeline _ _ [x] -> getExpr x + T_Redirecting _ _ c -> getExpr c + T_Condition _ _ c -> getExpr c + TC_Binary _ _ op lhs rhs -> orient (lhs, op, rhs) + _ -> Nothing + + -- Swap items so that the constant side is rhs (or Nothing if both/neither is constant) + orient (lhs, op, rhs) = + case (isConstant lhs, isConstant rhs) of + (True, False) -> return (rhs, op, lhs) + (False, True) -> return (lhs, op, rhs) + _ -> Nothing + + +checkAndEq _ _ = return () + + prop_checkValidCondOps1 = verify checkValidCondOps "[[ a -xz b ]]" prop_checkValidCondOps2 = verify checkValidCondOps "[ -M a ]" prop_checkValidCondOps2a = verifyNot checkValidCondOps "[ 3 \\> 2 ]" From 8ff0c5be7a85561d23aea762f0495c144896aee3 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 6 Apr 2025 19:26:54 -0700 Subject: [PATCH 448/459] Suppress SC2216 when piping to cp/mv/rm -i (fixes #3141). --- src/ShellCheck/Analytics.hs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 4757e57..86a3292 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -3596,6 +3596,8 @@ prop_checkPipeToNowhere17 = verify checkPipeToNowhere "echo World | cat << 'EOF' prop_checkPipeToNowhere18 = verifyNot checkPipeToNowhere "ls 1>&3 3>&1 3>&- | wc -l" prop_checkPipeToNowhere19 = verifyNot checkPipeToNowhere "find . -print0 | du --files0-from=/dev/stdin" prop_checkPipeToNowhere20 = verifyNot checkPipeToNowhere "find . | du --exclude-from=/dev/fd/0" +prop_checkPipeToNowhere21 = verifyNot checkPipeToNowhere "yes | cp -ri foo/* bar" +prop_checkPipeToNowhere22 = verifyNot checkPipeToNowhere "yes | rm --interactive *" data PipeType = StdoutPipe | StdoutStderrPipe | NoPipe deriving (Eq) checkPipeToNowhere :: Parameters -> Token -> WriterT [TokenComment] Identity () @@ -3661,6 +3663,7 @@ checkPipeToNowhere params t = commandSpecificException name cmd = case name of "du" -> any ((`elem` ["exclude-from", "files0-from"]) . snd) $ getAllFlags cmd + _ | name `elem` interactiveFlagCmds -> hasInteractiveFlag cmd _ -> False warnAboutDupes (n, list@(_:_:_)) = @@ -3684,7 +3687,7 @@ checkPipeToNowhere params t = name <- getCommandBasename cmd guard $ name `elem` nonReadingCommands guard . not $ hasAdditionalConsumers cmd - guard . not $ name `elem` ["cp", "mv", "rm"] && cmd `hasFlag` "i" + guard . not $ name `elem` interactiveFlagCmds && hasInteractiveFlag cmd let suggestion = if name == "echo" then "Did you want 'cat' instead?" @@ -3699,6 +3702,9 @@ checkPipeToNowhere params t = treeContains pred t = isNothing $ doAnalysis (guard . not . pred) t + interactiveFlagCmds = [ "cp", "mv", "rm" ] + hasInteractiveFlag cmd = cmd `hasFlag` "i" || cmd `hasFlag` "interactive" + mayConsume t = case t of T_ProcSub _ "<" _ -> True From 72af76f443b81bbcd10889df8caf19be1671f7a2 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sun, 6 Apr 2025 19:58:13 -0700 Subject: [PATCH 449/459] Supress SC2093 when execfail is set (fixes #3178) --- src/ShellCheck/Analytics.hs | 4 +++- src/ShellCheck/AnalyzerLib.hs | 6 ++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 86a3292..e6a1fd6 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1896,7 +1896,9 @@ prop_checkSpuriousExec8 = verifyNot checkSpuriousExec "exec {origout}>&1- >tmp.l prop_checkSpuriousExec9 = verify checkSpuriousExec "for file in rc.d/*; do exec \"$file\"; done" prop_checkSpuriousExec10 = verifyNot checkSpuriousExec "exec file; r=$?; printf >&2 'failed\n'; return $r" prop_checkSpuriousExec11 = verifyNot checkSpuriousExec "exec file; :" -checkSpuriousExec _ = doLists +prop_checkSpuriousExec12 = verifyNot checkSpuriousExec "#!/bin/bash\nshopt -s execfail; exec foo; exec bar; echo 'Error'; exit 1;" +prop_checkSpuriousExec13 = verify checkSpuriousExec "#!/bin/dash\nshopt -s execfail; exec foo; exec bar; echo 'Error'; exit 1;" +checkSpuriousExec params t = when (not $ hasExecfail params) $ doLists t where doLists (T_Script _ _ cmds) = doList cmds False doLists (T_BraceGroup _ cmds) = doList cmds False diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 531ce8b..da528a4 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -89,6 +89,8 @@ data Parameters = Parameters { hasSetE :: Bool, -- Whether this script has 'set -o pipefail' anywhere. hasPipefail :: Bool, + -- Whether this script has 'shopt -s execfail' anywhere. + hasExecfail :: Bool, -- A linear (bad) analysis of data flow variableFlow :: [StackData], -- A map from Id to Token @@ -226,6 +228,10 @@ makeParameters spec = params BusyboxSh -> isOptionSet "pipefail" root Sh -> True Ksh -> isOptionSet "pipefail" root, + hasExecfail = + case shellType params of + Bash -> isOptionSet "execfail" root + _ -> False, shellTypeSpecified = isJust (asShellType spec) || isJust (asFallbackShell spec), idMap = getTokenMap root, parentMap = getParentTree root, From e4853af5b0f541d8070d9c76adb59ccd9b1b44f0 Mon Sep 17 00:00:00 2001 From: Eisuke Kawashima Date: Fri, 21 Mar 2025 18:49:06 +0900 Subject: [PATCH 450/459] doc: update man --- shellcheck.1.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/shellcheck.1.md b/shellcheck.1.md index b873e45..c768bfe 100644 --- a/shellcheck.1.md +++ b/shellcheck.1.md @@ -78,7 +78,7 @@ not warn at all, as `ksh` supports decimals in arithmetic contexts. : Don't try to look for .shellcheckrc configuration files. ---rcfile\ RCFILE +**--rcfile** *RCFILE* : Prefer the specified configuration file over searching for one in the default locations. @@ -317,7 +317,7 @@ Here is an example `.shellcheckrc`: disable=SC2236 If no `.shellcheckrc` is found in any of the parent directories, ShellCheck -will look in `~/.shellcheckrc` followed by the XDG config directory +will look in `~/.shellcheckrc` followed by the `$XDG_CONFIG_HOME` (usually `~/.config/shellcheckrc`) on Unix, or `%APPDATA%/shellcheckrc` on Windows. Only the first file found will be used. @@ -403,4 +403,4 @@ see https://gnu.org/licenses/gpl.html # SEE ALSO -sh(1) bash(1) +sh(1) bash(1) dash(1) ksh(1) From 574c6d18fbbae18b65b244ce2b37c3dced452a5a Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 8 Apr 2025 10:23:10 -0700 Subject: [PATCH 451/459] Suggest using test -e instead of -a (fixes #3174). --- CHANGELOG.md | 1 + src/ShellCheck/Analytics.hs | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 612068d..6309192 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ - SC2327/SC2328: Warn about capturing the output of redirected commands. - SC2329: Warn when (non-escaping) functions are never invoked. - SC2330: Warn about unsupported glob matches with [[ .. ]] in BusyBox. +- SC2331: Suggest using standard -e instead of unary -a in tests. - Precompiled binaries for Linux riscv64 (linux.riscv64) ### Changed - SC2002 about Useless Use Of Cat is now disabled by default. It can be diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index e6a1fd6..4984d44 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -204,6 +204,7 @@ nodeChecks = [ ,checkUnnecessaryParens ,checkPlusEqualsNumber ,checkExpansionWithRedirection + ,checkUnaryTestA ] optionalChecks = map fst optionalTreeChecks @@ -5098,6 +5099,15 @@ checkExpansionWithRedirection params t = err redirectId 2328 $ "This redirection takes output away from the command substitution" ++ if suggestTee then " (use tee to duplicate)." else "." +prop_checkUnaryTestA1 = verify checkUnaryTestA "[ -a foo ]" +prop_checkUnaryTestA2 = verify checkUnaryTestA "[ ! -a foo ]" +prop_checkUnaryTestA3 = verifyNot checkUnaryTestA "[ foo -a bar ]" +checkUnaryTestA params t = + case t of + TC_Unary id _ "-a" _ -> + styleWithFix id 2331 "For file existence, prefer standard -e over legacy -a." $ + fixWith [replaceStart id params 2 "-e"] + _ -> return () return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) From c41f3a4b8ac4eb7bcff230928a47f8f92f15f49d Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 8 Apr 2025 10:53:52 -0700 Subject: [PATCH 452/459] Warn about [ ! -o opt ] (and -a) being unconditionally true (fixes #3174) --- CHANGELOG.md | 2 ++ src/ShellCheck/Checks/ShellSupport.hs | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6309192..bc646d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ - SC2329: Warn when (non-escaping) functions are never invoked. - SC2330: Warn about unsupported glob matches with [[ .. ]] in BusyBox. - SC2331: Suggest using standard -e instead of unary -a in tests. +- SC2332: Warn about `[ ! -o opt ]` being unconditionally true in Bash. +- SC3062: Warn about bashism `[ -o opt ]`. - Precompiled binaries for Linux riscv64 (linux.riscv64) ### Changed - SC2002 about Useless Use Of Cat is now disabled by default. It can be diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index f228832..624d474 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -63,6 +63,7 @@ checks = [ ,checkPS1Assignments ,checkMultipleBangs ,checkBangAfterPipe + ,checkNegatedUnaryOps ] testChecker (ForShell _ t) = @@ -218,6 +219,7 @@ prop_checkBashisms124 = verify checkBashisms "#!/bin/dash\ntype -p test" prop_checkBashisms125 = verifyNot checkBashisms "#!/bin/busybox sh\ntype -p test" prop_checkBashisms126 = verifyNot checkBashisms "#!/bin/busybox sh\nread -p foo -r bar" prop_checkBashisms127 = verifyNot checkBashisms "#!/bin/busybox sh\necho -ne foo" +prop_checkBashisms128 = verify checkBashisms "#!/bin/dash\ntype -p test" checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do params <- ask kludge params t @@ -272,6 +274,8 @@ checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do warnMsg id 3016 "unary -v (in place of [ -n \"${var+x}\" ]) is" bashism (TC_Unary id _ "-a" _) = warnMsg id 3017 "unary -a in place of -e is" + bashism (TC_Unary id _ "-o" _) = + warnMsg id 3062 "unary -o to check options is" bashism (T_SimpleCommand id _ [asStr -> Just "test", asStr -> Just "-a", _]) = warnMsg id 3017 "unary -a in place of -e is" bashism (TA_Unary id op _) @@ -649,5 +653,22 @@ checkBangAfterPipe = ForShell [Dash, BusyboxSh, Sh, Bash] f err id 2326 "! is not allowed in the middle of pipelines. Use command group as in cmd | { ! cmd; } if necessary." _ -> return () + +prop_checkNegatedUnaryOps1 = verify checkNegatedUnaryOps "[ ! -o braceexpand ]" +prop_checkNegatedUnaryOps2 = verifyNot checkNegatedUnaryOps "[ -o braceexpand ]" +prop_checkNegatedUnaryOps3 = verifyNot checkNegatedUnaryOps "[[ ! -o braceexpand ]]" +prop_checkNegatedUnaryOps4 = verifyNot checkNegatedUnaryOps "! [ -o braceexpand ]" +prop_checkNegatedUnaryOps5 = verify checkNegatedUnaryOps "[ ! -a file ]" +checkNegatedUnaryOps = ForShell [Bash] f + where + f token = case token of + TC_Unary id SingleBracket "!" (TC_Unary _ _ op _) | op `elem` ["-a", "-o"] -> + err id 2332 $ msg op + _ -> return () + + msg "-o" = "[ ! -o opt ] is always true because -o becomes logical OR. Use [[ ]] or ! [ -o opt ]." + msg "-a" = "[ ! -a file ] is always true because -a becomes logical AND. Use -e instead." + msg _ = pleaseReport "unhandled negated unary message" + return [] runTests = $( [| $(forAllProperties) (quickCheckWithResult (stdArgs { maxSuccess = 1 }) ) |]) From 7fc992d0dc590f32bd7265e719757102f5ad3b76 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 8 Apr 2025 20:52:52 -0700 Subject: [PATCH 453/459] Suppress SC2119/SC2120 for ${1:-default} (fixes #2023) --- src/ShellCheck/Analytics.hs | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 4984d44..85104d8 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2825,6 +2825,8 @@ prop_checkUnpassedInFunctions11 = verifyNotTree checkUnpassedInFunctions "foo() prop_checkUnpassedInFunctions12 = verifyNotTree checkUnpassedInFunctions "foo() { echo ${!var*}; }; foo;" prop_checkUnpassedInFunctions13 = verifyNotTree checkUnpassedInFunctions "# shellcheck disable=SC2120\nfoo() { echo $1; }\nfoo\n" prop_checkUnpassedInFunctions14 = verifyTree checkUnpassedInFunctions "foo() { echo $#; }; foo" +prop_checkUnpassedInFunctions15 = verifyNotTree checkUnpassedInFunctions "foo() { echo ${1-x}; }; foo" +prop_checkUnpassedInFunctions16 = verifyNotTree checkUnpassedInFunctions "foo() { echo ${1:-x}; }; foo" checkUnpassedInFunctions params root = execWriter $ mapM_ warnForGroup referenceGroups where @@ -2841,9 +2843,10 @@ checkUnpassedInFunctions params root = case x of Assignment (_, _, str, _) -> isPositional str _ -> False + isPositionalReference function x = case x of - Reference (_, t, str) -> isPositional str && t `isDirectChildOf` function + Reference (_, t, str) -> isPositional str && t `isDirectChildOf` function && not (hasDefaultValue t) _ -> False isDirectChildOf child parent = fromMaybe False $ do @@ -2857,6 +2860,7 @@ checkUnpassedInFunctions params root = referenceList :: [(String, Bool, Token)] referenceList = execWriter $ doAnalysis (sequence_ . checkCommand) root + checkCommand :: Token -> Maybe (Writer [(String, Bool, Token)] ()) checkCommand t@(T_SimpleCommand _ _ (cmd:args)) = do str <- getLiteralString cmd @@ -2867,6 +2871,21 @@ checkUnpassedInFunctions params root = isPositional str = str == "*" || str == "@" || str == "#" || (all isDigit str && str /= "0" && str /= "") + -- True if t is a variable that specifies a default value, + -- such as ${1-x} or ${1:-x}. + hasDefaultValue t = + case t of + T_DollarBraced _ True l -> + let str = concat $ oversimplify l + in isDefaultValueModifier $ getBracedModifier str + _ -> False + + isDefaultValueModifier str = + case str of + '-':_ -> True + ':':'-':_ -> True + _ -> False + isArgumentless (_, b, _) = b referenceGroups = Map.elems $ foldr updateWith Map.empty referenceList updateWith x@(name, _, _) = Map.insertWith (++) name [x] From 553a80f77ad6426107fe5fefcdc950e64c420b1d Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 8 Apr 2025 21:21:50 -0700 Subject: [PATCH 454/459] Also ignore SC2119 for :? and :+. --- src/ShellCheck/Analytics.hs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 85104d8..9eec8ed 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -2827,6 +2827,8 @@ prop_checkUnpassedInFunctions13 = verifyNotTree checkUnpassedInFunctions "# shel prop_checkUnpassedInFunctions14 = verifyTree checkUnpassedInFunctions "foo() { echo $#; }; foo" prop_checkUnpassedInFunctions15 = verifyNotTree checkUnpassedInFunctions "foo() { echo ${1-x}; }; foo" prop_checkUnpassedInFunctions16 = verifyNotTree checkUnpassedInFunctions "foo() { echo ${1:-x}; }; foo" +prop_checkUnpassedInFunctions17 = verifyNotTree checkUnpassedInFunctions "foo() { mycommand ${1+--verbose}; }; foo" +prop_checkUnpassedInFunctions18 = verifyNotTree checkUnpassedInFunctions "foo() { if mycheck; then foo ${1?Missing}; fi; }; foo" checkUnpassedInFunctions params root = execWriter $ mapM_ warnForGroup referenceGroups where @@ -2882,9 +2884,10 @@ checkUnpassedInFunctions params root = isDefaultValueModifier str = case str of - '-':_ -> True - ':':'-':_ -> True + ':':c:_ -> c `elem` handlesDefault + c:_ -> c `elem` handlesDefault _ -> False + where handlesDefault = "-+?" isArgumentless (_, b, _) = b referenceGroups = Map.elems $ foldr updateWith Map.empty referenceList From efb5a5a2741e690d43d5539809326d5e249fd9f2 Mon Sep 17 00:00:00 2001 From: Eisuke Kawashima Date: Tue, 25 Mar 2025 01:31:06 +0900 Subject: [PATCH 455/459] fix(SC3013): check POSIX-compliant unary operators for test and [ fix #2125 --- src/ShellCheck/Checks/ShellSupport.hs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 624d474..2039483 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -220,6 +220,9 @@ prop_checkBashisms125 = verifyNot checkBashisms "#!/bin/busybox sh\ntype -p test prop_checkBashisms126 = verifyNot checkBashisms "#!/bin/busybox sh\nread -p foo -r bar" prop_checkBashisms127 = verifyNot checkBashisms "#!/bin/busybox sh\necho -ne foo" prop_checkBashisms128 = verify checkBashisms "#!/bin/dash\ntype -p test" +prop_checkBashisms129 = verify checkBashisms "#!/bin/sh\n[ -k /tmp ]" +prop_checkBashisms130 = verifyNot checkBashisms "#!/bin/dash\ntest -k /tmp" +prop_checkBashisms131 = verify checkBashisms "#!/bin/sh\n[ -o errexit ]" checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do params <- ask kludge params t @@ -254,6 +257,18 @@ checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do bashism (T_SimpleCommand id _ [asStr -> Just "test", lhs, asStr -> Just op, rhs]) | op `elem` [ "<", ">", "\\<", "\\>", "<=", ">=", "\\<=", "\\>="] = unless isDash $ warnMsg id 3012 $ "lexicographical " ++ op ++ " is" + bashism (TC_Unary id _ op _) + | op `elem` [ "-k", "-G", "-O" ] = + unless isDash $ warnMsg id 3013 $ op ++ " is" + bashism (T_SimpleCommand id _ [asStr -> Just "test", asStr -> Just op, _]) + | op `elem` [ "-k", "-G", "-O" ] = + unless isDash $ warnMsg id 3013 $ op ++ " is" + bashism (TC_Unary id _ op _) + | op `elem` [ "-N", "-o", "-R" ] = + warnMsg id 3013 $ op ++ " is" + bashism (T_SimpleCommand id _ [asStr -> Just "test", asStr -> Just op, _]) + | op `elem` [ "-N", "-o", "-R" ] = + warnMsg id 3013 $ op ++ " is" bashism (TC_Binary id SingleBracket op _ _) | op `elem` [ "-ot", "-nt", "-ef" ] = unless isDash $ warnMsg id 3013 $ op ++ " is" From dc41f0cc5bdcf1c814b184892b20ee0d2822e95a Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Fri, 11 Apr 2025 14:14:09 -0700 Subject: [PATCH 456/459] Refactor checks for POSIX test flags --- src/ShellCheck/Checks/ShellSupport.hs | 98 +++++++++++++++------------ 1 file changed, 56 insertions(+), 42 deletions(-) diff --git a/src/ShellCheck/Checks/ShellSupport.hs b/src/ShellCheck/Checks/ShellSupport.hs index 2039483..c828555 100644 --- a/src/ShellCheck/Checks/ShellSupport.hs +++ b/src/ShellCheck/Checks/ShellSupport.hs @@ -251,48 +251,16 @@ checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do bashism (T_Condition id DoubleBracket _) = unless isBusyboxSh $ warnMsg id 3010 "[[ ]] is" bashism (T_HereString id _) = warnMsg id 3011 "here-strings are" - bashism (TC_Binary id SingleBracket op _ _) - | op `elem` [ "<", ">", "\\<", "\\>", "<=", ">=", "\\<=", "\\>="] = - unless isDash $ warnMsg id 3012 $ "lexicographical " ++ op ++ " is" - bashism (T_SimpleCommand id _ [asStr -> Just "test", lhs, asStr -> Just op, rhs]) - | op `elem` [ "<", ">", "\\<", "\\>", "<=", ">=", "\\<=", "\\>="] = - unless isDash $ warnMsg id 3012 $ "lexicographical " ++ op ++ " is" - bashism (TC_Unary id _ op _) - | op `elem` [ "-k", "-G", "-O" ] = - unless isDash $ warnMsg id 3013 $ op ++ " is" - bashism (T_SimpleCommand id _ [asStr -> Just "test", asStr -> Just op, _]) - | op `elem` [ "-k", "-G", "-O" ] = - unless isDash $ warnMsg id 3013 $ op ++ " is" - bashism (TC_Unary id _ op _) - | op `elem` [ "-N", "-o", "-R" ] = - warnMsg id 3013 $ op ++ " is" - bashism (T_SimpleCommand id _ [asStr -> Just "test", asStr -> Just op, _]) - | op `elem` [ "-N", "-o", "-R" ] = - warnMsg id 3013 $ op ++ " is" - bashism (TC_Binary id SingleBracket op _ _) - | op `elem` [ "-ot", "-nt", "-ef" ] = - unless isDash $ warnMsg id 3013 $ op ++ " is" - bashism (T_SimpleCommand id _ [asStr -> Just "test", lhs, asStr -> Just op, rhs]) - | op `elem` [ "-ot", "-nt", "-ef" ] = - unless isDash $ warnMsg id 3013 $ op ++ " is" - bashism (TC_Binary id SingleBracket "==" _ _) = - unless isBusyboxSh $ warnMsg id 3014 "== in place of = is" - bashism (T_SimpleCommand id _ [asStr -> Just "test", lhs, asStr -> Just "==", rhs]) = - unless isBusyboxSh $ warnMsg id 3014 "== in place of = is" - bashism (TC_Binary id SingleBracket "=~" _ _) = - warnMsg id 3015 "=~ regex matching is" - bashism (T_SimpleCommand id _ [asStr -> Just "test", lhs, asStr -> Just "=~", rhs]) = - warnMsg id 3015 "=~ regex matching is" - bashism (TC_Unary id SingleBracket "-v" _) = - warnMsg id 3016 "unary -v (in place of [ -n \"${var+x}\" ]) is" - bashism (T_SimpleCommand id _ [asStr -> Just "test", asStr -> Just "-v", _]) = - warnMsg id 3016 "unary -v (in place of [ -n \"${var+x}\" ]) is" - bashism (TC_Unary id _ "-a" _) = - warnMsg id 3017 "unary -a in place of -e is" - bashism (TC_Unary id _ "-o" _) = - warnMsg id 3062 "unary -o to check options is" - bashism (T_SimpleCommand id _ [asStr -> Just "test", asStr -> Just "-a", _]) = - warnMsg id 3017 "unary -a in place of -e is" + + bashism (TC_Binary id _ op _ _) = + checkTestOp bashismBinaryTestFlags op id + bashism (T_SimpleCommand id _ [asStr -> Just "test", lhs, asStr -> Just op, rhs]) = + checkTestOp bashismBinaryTestFlags op id + bashism (TC_Unary id _ op _) = + checkTestOp bashismUnaryTestFlags op id + bashism (T_SimpleCommand id _ [asStr -> Just "test", asStr -> Just op, _]) = + checkTestOp bashismUnaryTestFlags op id + bashism (TA_Unary id op _) | op `elem` [ "|++", "|--", "++|", "--|"] = warnMsg id 3018 $ filter (/= '|') op ++ " is" @@ -529,6 +497,52 @@ checkBashisms = ForShell [Sh, Dash, BusyboxSh] $ \t -> do Assignment (_, _, name, _) -> name == var _ -> False + checkTestOp table op id = sequence_ $ do + (code, shells, msg) <- Map.lookup op table + guard . not $ shellType params `elem` shells + return $ warnMsg id code (msg op) + + +buildTestFlagMap list = Map.fromList $ concatMap (\(x,y) -> map (\c -> (c,y)) x) list +bashismBinaryTestFlags = buildTestFlagMap [ + -- ([list of applicable flags], + -- (error code, exempt shells, message builder :: String -> String)), + -- + -- Distinct error codes allow the wiki to give more helpful, targeted + -- information. + (["<", ">", "\\<", "\\>", "<=", ">=", "\\<=", "\\>="], + (3012, [Dash, BusyboxSh], \op -> "lexicographical " ++ op ++ " is")), + (["-nt", "-ot", "-ef"], + (3013, [Dash, BusyboxSh], \op -> op ++ " is")), + (["=="], + (3014, [BusyboxSh], \op -> op ++ " in place of = is")), + (["=~"], + (3015, [], \op -> op ++ " regex matching is")), + + ([], (0,[],const "")) + ] +bashismUnaryTestFlags = buildTestFlagMap [ + (["-v"], + (3016, [], \op -> "test " ++ op ++ " (in place of [ -n \"${var+x}\" ]) is")), + (["-a"], + (3017, [], \op -> "unary " ++ op ++ " in place of -e is")), + (["-o"], + (3062, [], \op -> "test " ++ op ++ " to check options is")), + (["-R"], + (3063, [], \op -> "test " ++ op ++ " and namerefs in general are")), + (["-N"], + (3064, [], \op -> "test " ++ op ++ " is")), + (["-k"], + (3065, [Dash, BusyboxSh], \op -> "test " ++ op ++ " is")), + (["-G"], + (3066, [Dash, BusyboxSh], \op -> "test " ++ op ++ " is")), + (["-O"], + (3067, [Dash, BusyboxSh], \op -> "test " ++ op ++ " is")), + + ([], (0,[],const "")) + ] + + prop_checkEchoSed1 = verify checkEchoSed "FOO=$(echo \"$cow\" | sed 's/foo/bar/g')" prop_checkEchoSed1b = verify checkEchoSed "FOO=$(sed 's/foo/bar/g' <<< \"$cow\")" prop_checkEchoSed2 = verify checkEchoSed "rm $(echo $cow | sed -e 's,foo,bar,')" From f78714e0f6070bc4efa2f7c11c1ad632e8c250a2 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Fri, 11 Apr 2025 19:14:53 -0700 Subject: [PATCH 457/459] Add ":" alongside "true" for SC2015 --- src/ShellCheck/Analytics.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 5f8c84c..ac686ff 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -890,7 +890,7 @@ checkShorthandIf params x@(T_OrIf _ (T_AndIf id _ b) (T_Pipeline _ _ t)) where isOk [t] = isAssignment t || fromMaybe False (do name <- getCommandBasename t - return $ name `elem` ["echo", "exit", "return", "printf", "true"]) + return $ name `elem` ["echo", "exit", "return", "printf", "true", ":"]) isOk _ = False inCondition = isCondition $ getPath (parentMap params) x checkShorthandIf _ _ = return () From b381658dbc74ebe7141717f7723be3a8b39121f9 Mon Sep 17 00:00:00 2001 From: Ian Ehrenwald Date: Fri, 25 Apr 2025 14:11:07 -0400 Subject: [PATCH 458/459] Add python3 to the list of badShells --- src/ShellCheck/Parser.hs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index d019d89..84c3ce4 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -3397,6 +3397,7 @@ readScriptFile sourced = do "fish", "perl", "python", + "python3", "ruby", "tcsh", "zsh" From 47d358c1d44a84a1d54a4118fba4cf7668a9225d Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Sat, 17 May 2025 00:55:50 +0000 Subject: [PATCH 459/459] Tighten SC2333/SC2334 to only trigger against literals. --- src/ShellCheck/ASTLib.hs | 6 ++++++ src/ShellCheck/Analytics.hs | 29 +++++++++++++++++++---------- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 6b26b22..1e1b9cd 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -446,6 +446,12 @@ getLiteralStringExt more = g -- Is this token a string literal? isLiteral t = isJust $ getLiteralString t +-- Is this token a string literal number? +isLiteralNumber t = fromMaybe False $ do + s <- getLiteralString t + guard $ all isDigit s + return True + -- Escape user data for messages. -- Messages generally avoid repeating user data, but sometimes it's helpful. e4m = escapeForMessage diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 073e911..2e9a3bd 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1635,8 +1635,8 @@ checkOrNeq _ (T_OrIf id lhs rhs) = sequence_ $ do checkOrNeq _ _ = return () -prop_checkAndEq1 = verify checkAndEq "if [[ $lol -eq cow && $lol -eq foo ]]; then echo foo; fi" -prop_checkAndEq2 = verify checkAndEq "(( a==lol && a==foo ))" +prop_checkAndEq1 = verifyNot checkAndEq "cow=0; foo=0; if [[ $lol -eq cow && $lol -eq foo ]]; then echo foo; fi" +prop_checkAndEq2 = verifyNot checkAndEq "lol=0 foo=0; (( a==lol && a==foo ))" prop_checkAndEq3 = verify checkAndEq "[ \"$a\" = lol && \"$a\" = foo ]" prop_checkAndEq4 = verifyNot checkAndEq "[ a = $cow && b = $foo ]" prop_checkAndEq5 = verifyNot checkAndEq "[[ $a = /home && $a = */public_html/* ]]" @@ -1644,25 +1644,34 @@ prop_checkAndEq6 = verify checkAndEq "[ $a = a ] && [ $a = b ]" prop_checkAndEq7 = verify checkAndEq "[ $a = a ] && [ $a = b ] || true" prop_checkAndEq8 = verifyNot checkAndEq "[[ $a == x && $a == x ]]" prop_checkAndEq9 = verifyNot checkAndEq "[ 0 -eq $FOO ] && [ 0 -eq $BAR ]" +prop_checkAndEq10 = verify checkAndEq "(( a == 1 && a == 2 ))" +prop_checkAndEq11 = verify checkAndEq "[ $x -eq 1 ] && [ $x -eq 2 ]" +prop_checkAndEq12 = verify checkAndEq "[ 1 -eq $x ] && [ $x -eq 2 ]" +prop_checkAndEq13 = verifyNot checkAndEq "[ 1 -eq $x ] && [ $x -eq 1 ]" +prop_checkAndEq14 = verifyNot checkAndEq "[ $a = $b ] && [ $a = $c ]" + +checkAndEqOperands "-eq" rhs1 rhs2 = isLiteralNumber rhs1 && isLiteralNumber rhs2 +checkAndEqOperands op rhs1 rhs2 | op == "=" || op == "==" = isLiteral rhs1 && isLiteral rhs2 +checkAndEqOperands _ _ _ = False -- For test-level "and": [ x = y -a x = z ] checkAndEq _ (TC_And id typ op (TC_Binary _ _ op1 lhs1 rhs1 ) (TC_Binary _ _ op2 lhs2 rhs2)) - | (op1 == op2 && (op1 == "-eq" || op1 == "=" || op1 == "==")) && lhs1 == lhs2 && rhs1 /= rhs2 && not (any isGlob [rhs1,rhs2]) = - warn id 2055 $ "You probably wanted " ++ (if typ == SingleBracket then "-o" else "||") ++ " here, otherwise it's always false." + | op1 == op2 && lhs1 == lhs2 && rhs1 /= rhs2 && checkAndEqOperands op1 rhs1 rhs2 = + warn id 2333 $ "You probably wanted " ++ (if typ == SingleBracket then "-o" else "||") ++ " here, otherwise it's always false." -- For arithmetic context "and" -checkAndEq _ (TA_Binary id "&&" (TA_Binary _ "==" word1 _) (TA_Binary _ "==" word2 _)) - | word1 == word2 = - warn id 2056 "You probably wanted || here, otherwise it's always false." +checkAndEq _ (TA_Binary id "&&" (TA_Binary _ "==" lhs1 rhs1) (TA_Binary _ "==" lhs2 rhs2)) + | lhs1 == lhs2 && isLiteralNumber rhs1 && isLiteralNumber rhs2 = + warn id 2334 "You probably wanted || here, otherwise it's always false." -- For command level "and": [ x = y ] && [ x = z ] checkAndEq _ (T_AndIf id lhs rhs) = sequence_ $ do (lhs1, op1, rhs1) <- getExpr lhs (lhs2, op2, rhs2) <- getExpr rhs - guard $ op1 == op2 && op1 `elem` ["-eq", "=", "=="] + guard $ op1 == op2 guard $ lhs1 == lhs2 && rhs1 /= rhs2 - guard . not $ any isGlob [rhs1, rhs2] - return $ warn id 2252 "You probably wanted || here, otherwise it's always false." + guard $ checkAndEqOperands op1 rhs1 rhs2 + return $ warn id 2333 "You probably wanted || here, otherwise it's always false." where getExpr x = case x of