From d1f25f3b4dde7c44bcaa72d2691a91204564ff78 Mon Sep 17 00:00:00 2001 From: Raman Gupta Date: Thu, 22 Jun 2023 08:35:34 -0400 Subject: [PATCH 1/8] Fix and expand manual tests with auth repo --- test/TestsReadme.md | 36 ++++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/test/TestsReadme.md b/test/TestsReadme.md index bdaa493a..163e3d8d 100644 --- a/test/TestsReadme.md +++ b/test/TestsReadme.md @@ -160,11 +160,11 @@ As of this writing, testing the credentials is only done manually with a dockeri docker run --name artifactory -d -p 8081:8081 docker.bintray.io/jfrog/artifactory-oss:latest # Copy preconfigured gloabl config (with custom repo) and security config (with credentials user) into container. -docker cp ./test/resources/artifactory_config/artifactory.config.xml artifactory:/var/opt/jfrog/artifactory/etc/artifactory.config.import.xml -docker cp ./test/resources/artifactory_config/security_descriptor.xml artifactory:/var/opt/jfrog/artifactory/etc/security.import.xml +docker cp ./test/resources/artifactory_config/artifactory.config.xml artifactory:/var/opt/jfrog/artifactory/etc/artifactory/artifactory.config.import.xml +docker cp ./test/resources/artifactory_config/security_descriptor.xml artifactory:/var/opt/jfrog/artifactory/etc/artifactory/security.import.xml # Make the configs accessable -docker exec -u 0 -it artifactory sh -c 'chmod 777 $ARTIFACTORY_HOME/etc/*.import.xml' +docker exec -u 0 -it artifactory sh -c 'chmod 777 /var/opt/jfrog/artifactory/etc/artifactory/*.import.xml' # Restart docker after is done with initial booting (otherwise restart breaks the container). echo "sleeping for 15..." && sleep 15 @@ -189,7 +189,35 @@ echo ' @file:DependsOn("com.jcabi:jcabi-aether:0.10.1") // If unencrypted works via jcenter @file:DependsOnMaven("group:somejar:1.0") // If encrypted works. println("Hello, World!") -' | kscript - +' | kscript -c - +``` + +#### 4. Test environment variable substitution + +```bash +export auth_user="auth_user" +export auth_password="password" + +echo ' +@file:Repository("http://localhost:8081/artifactory/authenticated_repo", user="{{auth_user}}", password="{{auth_password}}") +@file:DependsOn("com.jcabi:jcabi-aether:0.10.1") // If unencrypted works via jcenter +@file:DependsOnMaven("group:somejar:1.0") // If encrypted works. +println("Hello, World!") +' | kscript -c - +``` + +#### 5. Test environment variable substitution with packaging + +```bash +export auth_user="auth_user" +export auth_password="password" + +echo ' +@file:Repository("http://localhost:8081/artifactory/authenticated_repo", user="{{auth_user}}", password="{{auth_password}}") +@file:DependsOn("com.jcabi:jcabi-aether:0.10.1") // If unencrypted works via jcenter +@file:DependsOnMaven("group:somejar:1.0") // If encrypted works. +println("Hello, World!") +' | kscript -c --package - ``` ### Additional info for manual testing From 7c2cc8cccd06ca56f79e3593da679f1851671a4c Mon Sep 17 00:00:00 2001 From: Raman Gupta Date: Thu, 22 Jun 2023 09:03:12 -0400 Subject: [PATCH 2/8] Fix docs for env var repo user/password substitution --- README.adoc | 18 +++++++++--------- test/TestsReadme.md | 10 ++++++++-- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/README.adoc b/README.adoc index 73377af0..8832dc4b 100644 --- a/README.adoc +++ b/README.adoc @@ -139,7 +139,7 @@ Two workarounds for ArchLinux exists, which can be used to make 'kscript' workin [source,bash] ---- sudo mkdir /usr/share/kotlin/bin -sudo ln -s /usr/bin/kotlin /usr/share/kotlin/bin/kotlin +sudo ln -s /usr/bin/kotlin /usr/share/kotlin/bin/kotlin sudo ln -s /usr/bin/kotlinc /usr/share/kotlin/bin/kotlinc ---- @@ -243,7 +243,7 @@ EOF [source,bash] ---- -kscript <(echo 'println("k-onliner")') arg1 arg2 arg3 +kscript <(echo 'println("k-onliner")') arg1 arg2 arg3 ---- Inlined _kscripts_ are also cached based on `md5` checksum, so running the same snippet again will use a cached jar ( @@ -259,7 +259,7 @@ script for better readability) [source,bash] ---- -kscript https://git.io/v1cG6 my argu ments +kscript https://git.io/v1cG6 my argu ments ---- To streamline the usage, the first part could be even aliased: @@ -414,12 +414,12 @@ The latter is the default for `kt` files and could be omitted @file:DependsOnMaven("net.clearvolume:cleargl:2.0.1") // Note that for compatibility reasons, only one locator argument is allowed for @DependsOnMaven -// also protected artifact repositories are supported, see +// also protected repositories are supported, see // @file:Repository("my-art", "http://localhost:8081/artifactory/authenticated_repo", user="auth_user", password="password") -// You can use environment variables for user and password when string surrounded by double {} brackets -// @file:Repository("my-art", "http://localhost:8081/artifactory/authenticated_repo", user="{{ARTIFACTORY_USER}}", password="{{ARTIFACTORY_PASSWORD}}") +// You can use environment variables for user and password +// @file:Repository("my-art", "http://localhost:8081/artifactory/authenticated_repo", user="$ARTIFACTORY_USER", password="$ARTIFACTORY_PASSWORD") // will be use 'ARTIFACTORY_USER' and 'ARTIFACTORY_PASSWORD' environment variables -// if the value doesn't found in the script environment will fail +// if a referenced environment variable is not found, the script will fail // Include helper scripts without deployment or prior compilation @file:Import("util.kt") @@ -516,7 +516,7 @@ which will bring up the classpath-enhanced REPL: Creating REPL from CountRecords.kts Welcome to Kotlin version 1.1.51 (JRE 1.8.0_151-b12) >>> import de.mpicbg.scicomp.bioinfo.openFasta ->>> +>>> ---- == Boostrap IDEA from a scriptlet @@ -582,7 +582,7 @@ KScript follows XDG directory standard, so the file should be created in (paths existing path is used): |=== -|OS |PATH +|OS |PATH |*Windows* | %LOCALAPPDATA%\kscript\kscript.properties; %USERPROFILE%.config\kscript\kscript.properties |*MacOs* | ~/Library/Application Support/kscript/kscript.properties; diff --git a/test/TestsReadme.md b/test/TestsReadme.md index 163e3d8d..fe687a99 100644 --- a/test/TestsReadme.md +++ b/test/TestsReadme.md @@ -199,7 +199,7 @@ export auth_user="auth_user" export auth_password="password" echo ' -@file:Repository("http://localhost:8081/artifactory/authenticated_repo", user="{{auth_user}}", password="{{auth_password}}") +@file:Repository("http://localhost:8081/artifactory/authenticated_repo", user="$auth_user", password="$auth_password") @file:DependsOn("com.jcabi:jcabi-aether:0.10.1") // If unencrypted works via jcenter @file:DependsOnMaven("group:somejar:1.0") // If encrypted works. println("Hello, World!") @@ -213,13 +213,19 @@ export auth_user="auth_user" export auth_password="password" echo ' -@file:Repository("http://localhost:8081/artifactory/authenticated_repo", user="{{auth_user}}", password="{{auth_password}}") +@file:Repository("http://localhost:8081/artifactory/authenticated_repo", user="$auth_user", password="$auth_password") @file:DependsOn("com.jcabi:jcabi-aether:0.10.1") // If unencrypted works via jcenter @file:DependsOnMaven("group:somejar:1.0") // If encrypted works. println("Hello, World!") ' | kscript -c --package - ``` +#### 6. (Optional) Cleanup Local Maven Cache + +```bash +rm -fr ~/.m2/repository/group +``` + ### Additional info for manual testing - Docker & container docu: https://www.jfrog.com/confluence/display/RTF/Installing+with+Docker From 416e6869d57f1abc8b037ec006c15f6311e9d44a Mon Sep 17 00:00:00 2001 From: Raman Gupta Date: Thu, 22 Jun 2023 09:07:59 -0400 Subject: [PATCH 3/8] Add NEWS for env var change --- NEWS.adoc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/NEWS.adoc b/NEWS.adoc index 09148c36..fcbe416c 100644 --- a/NEWS.adoc +++ b/NEWS.adoc @@ -1,5 +1,11 @@ = Changes +== [Future] - ? + +=== Fixed + +* (*Breaking*) Since migrating to a Kotlin Scripting backend, environment variables for repository authentication now use the `$FOO` syntax vs the `{{FOO}}` syntax. This has been broken since 4.2.0. (fixed thanks to https://github.com/rocketraman[rocketraman]) + == [4.2.2] - 2023-04-29 === Added From abe74eb02aec2ca4d85f04c92ceb21a780ff355c Mon Sep 17 00:00:00 2001 From: Raman Gupta Date: Thu, 22 Jun 2023 09:37:44 -0400 Subject: [PATCH 4/8] Fix packaging with repo env var substitutions --- NEWS.adoc | 1 + .../kscript/code/GradleTemplates.kt | 31 +++++++++++++++++-- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/NEWS.adoc b/NEWS.adoc index fcbe416c..bdda9e52 100644 --- a/NEWS.adoc +++ b/NEWS.adoc @@ -5,6 +5,7 @@ === Fixed * (*Breaking*) Since migrating to a Kotlin Scripting backend, environment variables for repository authentication now use the `$FOO` syntax vs the `{{FOO}}` syntax. This has been broken since 4.2.0. (fixed thanks to https://github.com/rocketraman[rocketraman]) +* Packaging scripts via `--package` did not work with repository authentication (thanks to https://github.com/rocketraman[rocketraman]) == [4.2.2] - 2023-04-29 diff --git a/src/main/kotlin/io/github/kscripting/kscript/code/GradleTemplates.kt b/src/main/kotlin/io/github/kscripting/kscript/code/GradleTemplates.kt index 56ad00a4..a1bc76c4 100644 --- a/src/main/kotlin/io/github/kscripting/kscript/code/GradleTemplates.kt +++ b/src/main/kotlin/io/github/kscripting/kscript/code/GradleTemplates.kt @@ -5,6 +5,9 @@ import io.github.kscripting.kscript.model.CompilerOpt import io.github.kscripting.kscript.model.Dependency import io.github.kscripting.kscript.model.Repository import io.github.kscripting.kscript.model.Script +import io.github.kscripting.kscript.util.Logger.errorMsg +import kotlin.onFailure +import kotlin.script.experimental.api.* object GradleTemplates { fun createGradleIdeaScript(script: Script): String { @@ -108,9 +111,15 @@ object GradleTemplates { private fun createGradleRepositoryCredentials(repository: Repository): String { if (repository.user.isNotBlank() && repository.password.isNotBlank()) { + fun getFinalValue(optionName: String, rawValue: String?): String? = + tryResolveEnvironmentVariable(rawValue, optionName) + + val username = getFinalValue("username", repository.user) + val password = getFinalValue("password", repository.password) + return """|credentials { - | username = "${repository.user}" - | password = "${repository.password}" + | username = "$username" + | password = "$password" |}""".trimMargin() } @@ -156,4 +165,22 @@ object GradleTemplates { | } |}""".trimMargin() } + + /** + * This is a variant of [kotlin.script.experimental.dependencies.maven.MavenDependenciesResolver.tryResolveEnvironmentVariable]. + */ + private fun tryResolveEnvironmentVariable( + str: String?, + optionName: String, + ): String? { + if (str == null) return null + if (!str.startsWith("$")) return str + val envName = str.substring(1) + val envValue: String? = System.getenv(envName) + if (envValue.isNullOrEmpty()) { + errorMsg("Environment variable '$envName' is not defined for option '$optionName'") + return null + } + return envValue + } } From df6defafd29262ee59c5b355101c2bb9f80f7e6b Mon Sep 17 00:00:00 2001 From: Raman Gupta Date: Thu, 22 Jun 2023 10:24:45 -0400 Subject: [PATCH 5/8] Fix jar_dependency for recent gradle --- test/resources/jar_dependency/build.gradle | 35 +++++++++++++++++-- .../jar_dependency/settings.gradle.kts | 1 + 2 files changed, 33 insertions(+), 3 deletions(-) create mode 100644 test/resources/jar_dependency/settings.gradle.kts diff --git a/test/resources/jar_dependency/build.gradle b/test/resources/jar_dependency/build.gradle index d633d136..33bb852d 100644 --- a/test/resources/jar_dependency/build.gradle +++ b/test/resources/jar_dependency/build.gradle @@ -1,6 +1,7 @@ plugins { id 'java' - id 'org.jetbrains.kotlin.jvm' version '1.3.41' + id 'org.jetbrains.kotlin.jvm' version '1.8.20' + id 'maven-publish' } group 'com.github.holgerbrandl.kscript.test' @@ -10,11 +11,21 @@ sourceCompatibility = 1.8 repositories { mavenCentral() + // for use with local artifactory for testing auth repo access, see artifactory_config and + // TestsReadme.md (Manual testing) + maven { + url "http://localhost:8081/artifactory/authenticated_repo" + allowInsecureProtocol = true + credentials { + username 'auth_user' + password 'password' + } + } } dependencies { implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8" - testCompile group: 'junit', name: 'junit', version: '4.12' + testImplementation group: 'junit', name: 'junit', version: '4.12' } compileKotlin { @@ -22,4 +33,22 @@ compileKotlin { } compileTestKotlin { kotlinOptions.jvmTarget = "1.8" -} \ No newline at end of file +} + +publishing { + publications { + mavenJava(MavenPublication) { + from components.java + } + } + repositories { + maven { + url "http://localhost:8081/artifactory/authenticated_repo" + allowInsecureProtocol = true + credentials { + username 'auth_user' + password 'password' + } + } + } +} diff --git a/test/resources/jar_dependency/settings.gradle.kts b/test/resources/jar_dependency/settings.gradle.kts new file mode 100644 index 00000000..36510d5f --- /dev/null +++ b/test/resources/jar_dependency/settings.gradle.kts @@ -0,0 +1 @@ +rootProject.name = "jartester" From cda51e4272d8a653ae06360d3149861f9ee00728 Mon Sep 17 00:00:00 2001 From: Raman Gupta Date: Thu, 22 Jun 2023 10:26:13 -0400 Subject: [PATCH 6/8] packaging: improve repo resolution order We resolve our custom repositories in advance of maven central. We also move maven local last -- its arguable whether this should even be here. --- .../io/github/kscripting/kscript/code/GradleTemplates.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/io/github/kscripting/kscript/code/GradleTemplates.kt b/src/main/kotlin/io/github/kscripting/kscript/code/GradleTemplates.kt index a1bc76c4..90ea9db1 100644 --- a/src/main/kotlin/io/github/kscripting/kscript/code/GradleTemplates.kt +++ b/src/main/kotlin/io/github/kscripting/kscript/code/GradleTemplates.kt @@ -64,9 +64,9 @@ object GradleTemplates { |} | |repositories { - | mavenLocal() - | mavenCentral() |${createGradleRepositoriesSection(script.repositories).prependIndent()} + | mavenCentral() + | mavenLocal() |} | |tasks.jar { From 50a3ce389fb0b914a563de63f667f03449867589 Mon Sep 17 00:00:00 2001 From: Raman Gupta Date: Thu, 22 Jun 2023 10:26:42 -0400 Subject: [PATCH 7/8] packaging: allow insecure repos If the script downloads from an insecure repo, then gradle should too. --- .../kotlin/io/github/kscripting/kscript/code/GradleTemplates.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/kotlin/io/github/kscripting/kscript/code/GradleTemplates.kt b/src/main/kotlin/io/github/kscripting/kscript/code/GradleTemplates.kt index 90ea9db1..ce41fc7b 100644 --- a/src/main/kotlin/io/github/kscripting/kscript/code/GradleTemplates.kt +++ b/src/main/kotlin/io/github/kscripting/kscript/code/GradleTemplates.kt @@ -133,6 +133,7 @@ object GradleTemplates { private fun createGradleRepositoriesSection(repositories: Set) = repositories.joinToString("\n") { """|maven { | url = uri("${it.url}") + | isAllowInsecureProtocol = true |${createGradleRepositoryCredentials(it).prependIndent()} |} """.trimMargin() From bfce680fd4419cccb8a4c4206a65f333dffa1ed3 Mon Sep 17 00:00:00 2001 From: Raman Gupta Date: Thu, 22 Jun 2023 10:27:43 -0400 Subject: [PATCH 8/8] test: fix manual test for gradle packaging We need to download a real package for the gradle packaging test,' otherwise Gradle complains. Publish the existing jar_dependency test and depend on that for the test. --- test/TestsReadme.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/TestsReadme.md b/test/TestsReadme.md index fe687a99..4f7ea91a 100644 --- a/test/TestsReadme.md +++ b/test/TestsReadme.md @@ -208,6 +208,12 @@ println("Hello, World!") #### 5. Test environment variable substitution with packaging +First publish a real package to our repository: + +```shell +gradle --project-dir test/resources/jar_dependency publishAllPublicationsToMavenRepository +``` + ```bash export auth_user="auth_user" export auth_password="password" @@ -215,7 +221,7 @@ export auth_password="password" echo ' @file:Repository("http://localhost:8081/artifactory/authenticated_repo", user="$auth_user", password="$auth_password") @file:DependsOn("com.jcabi:jcabi-aether:0.10.1") // If unencrypted works via jcenter -@file:DependsOnMaven("group:somejar:1.0") // If encrypted works. +@file:DependsOnMaven("com.github.holgerbrandl.kscript.test:jartester:1.0-SNAPSHOT") // If encrypted works. println("Hello, World!") ' | kscript -c --package - ```