Support returning java_buildpack version from droplets api

[xyloman]

In order to determine if an application needs to restage due to a new java_buildpack the buildpack version associated to the droplet should be included.

▶ cf curl /v3/droplets/cd6b4027-e906-43f5-91d0-9520a41456d2
{
   "guid": "cd6b4027-e906-43f5-91d0-9520a41456d2",
   "state": "STAGED",
   "error": null,
   "lifecycle": {
      "type": "buildpack",
      "data": {}
   },
   "checksum": {
      "type": "sha256",
      "value": "33f3d86eeb9ca116640d640835fca32b2de8755355a717097622b57db041a29e"
   },
   "buildpacks": [
      {
         "name": "dotnet_core_buildpack",
         "detect_output": "dotnet-core",
         "buildpack_name": "dotnet-core",
         "version": "2.3.7"
      }
   ],
   "stack": "cflinuxfs3",
   "image": null,
   "execution_metadata": "",
   "process_types": {
      "web": "cd ${HOME} && exec ./SampleApp --server.urls http://0.0.0.0:${PORT}"
   },
   "created_at": "2020-04-20T21:49:23Z",
   "updated_at": "2020-04-20T21:49:31Z",
   "relationships": {
      "app": {
         "data": {
            "guid": "1963c827-f37e-4b39-a3c5-10a60750835c"
         }
      }
   },
   "links": {
      "self": {
         "href": "https://api.sys.test.region.iaas.foundation.io/v3/droplets/cd6b4027-e906-43f5-91d0-9520a41456d2"
      },
      "app": {
         "href": "https://api.sys.test.region.iaas.foundation.io/v3/apps/1963c827-f37e-4b39-a3c5-10a60750835c"
      },
      "assign_current_droplet": {
         "href": "https://api.sys.test.region.iaas.foundation.io/v3/apps/1963c827-f37e-4b39-a3c5-10a60750835c/relationships/current_droplet",
         "method": "PATCH"
      },
      "package": {
         "href": "https://api.sys.test.region.iaas.foundation.io/v3/packages/942e37ad-d458-4f3a-8061-3f56ebdb484c"
      }
   },
   "metadata": {
      "labels": {},
      "annotations": {}
   }
}
 
▶ cf curl /v3/droplets/746491bf-afb9-4533-b3c3-91c4f5564e9d
{
   "guid": "746491bf-afb9-4533-b3c3-91c4f5564e9d",
   "state": "STAGED",
   "error": null,
   "lifecycle": {
      "type": "buildpack",
      "data": {}
   },
   "checksum": {
      "type": "sha256",
      "value": "cb351a2300dd0744d95c07732b24c31d7c1252022eb8a63bc71e446ade8f54d4"
   },
   "buildpacks": [
      {
         "name": "java_buildpack_offline",
         "detect_output": "",
         "buildpack_name": "",
         "version": null
      }
   ],
   "stack": "cflinuxfs3",
   "image": null,
   "execution_metadata": "",
   "process_types": {
      "task": "JAVA_OPTS=\"-agentpath:$PWD/.java-buildpack/open_jdk_jre/bin/jvmkill-1.16.0_RELEASE=printHeapHistogram=1 -Djava.io.tmpdir=$TMPDIR -XX:ActiveProcessorCount=$(nproc) -Djava.ext.dirs=$PWD/.java-buildpack/container_security_provider:$PWD/.java-buildpack/open_jdk_jre/lib/ext -Djava.security.properties=$PWD/.java-buildpack/java_security/java.security $JAVA_OPTS\" && CALCULATED_MEMORY=$($PWD/.java-buildpack/open_jdk_jre/bin/java-buildpack-memory-calculator-3.13.0_RELEASE -totMemory=$MEMORY_LIMIT -loadedClasses=12485 -poolType=metaspace -stackThreads=250 -vmOptions=\"$JAVA_OPTS\") && echo JVM Memory Configuration: $CALCULATED_MEMORY && JAVA_OPTS=\"$JAVA_OPTS $CALCULATED_MEMORY\" && MALLOC_ARENA_MAX=2 SERVER_PORT=$PORT eval exec $PWD/.java-buildpack/open_jdk_jre/bin/java $JAVA_OPTS -cp $PWD/. org.springframework.boot.loader.JarLauncher",
      "web": "JAVA_OPTS=\"-agentpath:$PWD/.java-buildpack/open_jdk_jre/bin/jvmkill-1.16.0_RELEASE=printHeapHistogram=1 -Djava.io.tmpdir=$TMPDIR -XX:ActiveProcessorCount=$(nproc) -Djava.ext.dirs=$PWD/.java-buildpack/container_security_provider:$PWD/.java-buildpack/open_jdk_jre/lib/ext -Djava.security.properties=$PWD/.java-buildpack/java_security/java.security $JAVA_OPTS\" && CALCULATED_MEMORY=$($PWD/.java-buildpack/open_jdk_jre/bin/java-buildpack-memory-calculator-3.13.0_RELEASE -totMemory=$MEMORY_LIMIT -loadedClasses=12485 -poolType=metaspace -stackThreads=250 -vmOptions=\"$JAVA_OPTS\") && echo JVM Memory Configuration: $CALCULATED_MEMORY && JAVA_OPTS=\"$JAVA_OPTS $CALCULATED_MEMORY\" && MALLOC_ARENA_MAX=2 SERVER_PORT=$PORT eval exec $PWD/.java-buildpack/open_jdk_jre/bin/java $JAVA_OPTS -cp $PWD/. org.springframework.boot.loader.JarLauncher"
   },
   "created_at": "2020-04-20T21:48:31Z",
   "updated_at": "2020-04-20T21:48:36Z",
   "relationships": {
      "app": {
         "data": {
            "guid": "7bf1e72d-ec90-4ec5-99e3-21f94535fc73"
         }
      }
   },
   "links": {
      "self": {
         "href": "https://api.sys.test.region.iaas.foundation.io/v3/droplets/746491bf-afb9-4533-b3c3-91c4f5564e9d"
      },
      "app": {
         "href": "https://api.sys.test.region.iaas.foundation.io/v3/apps/7bf1e72d-ec90-4ec5-99e3-21f94535fc73"
      },
      "assign_current_droplet": {
         "href": "https://api.sys.test.region.iaas.foundation.io/v3/apps/7bf1e72d-ec90-4ec5-99e3-21f94535fc73/relationships/current_droplet",
         "method": "PATCH"
      },
      "package": {
         "href": "https://api.sys.test.region.iaas.foundation.io/v3/packages/03e68633-6966-45d7-819e-367dd20c085d"
      }
   },
   "metadata": {
      "labels": {},
      "annotations": {}
   }
}

▶ cf buildpacks
Getting buildpacks...

buildpack                      position   enabled   locked   filename                                             stack
appmetrics_nodejs8_buildpack   1          true      false    nodejs-buildpack-cflinuxfs3-v1.7.8.zip               cflinuxfs3
hwc_buildpack                  2          true      false    hwc_buildpack-cached-windows-v3.1.6.zip              windows
hwc_buildpack                  3          true      false    hwc_buildpack-cached-windows2016-v3.1.6.zip          windows2016
hwc_buildpack                  4          true      false    hwc_buildpack-cached-windows2012R2-v3.1.6.zip        windows2012R2
binary_buildpack               5          true      false    binary_buildpack-cached-windows2012R2-v1.0.36.zip    windows2012R2
binary_buildpack               6          true      false    binary_buildpack-cached-windows2016-v1.0.36.zip      windows2016
staticfile_buildpack           7          true      false    staticfile_buildpack-cached-cflinuxfs3-v1.5.5.zip    cflinuxfs3
java_buildpack_offline         8          true      false    java-buildpack-offline-cflinuxfs3-v4.29.1.zip        cflinuxfs3
ruby_buildpack                 9          true      false    ruby_buildpack-cached-cflinuxfs3-v1.8.14.zip         cflinuxfs3
nodejs_buildpack               10         true      false    nodejs_buildpack-cached-cflinuxfs3-v1.7.15.zip       cflinuxfs3
go_buildpack                   11         true      false    go_buildpack-cached-cflinuxfs3-v1.9.8.zip            cflinuxfs3
python_buildpack               12         true      false    python_buildpack-cached-cflinuxfs3-v1.7.10.zip       cflinuxfs3
php_buildpack                  13         true      false    php_buildpack-cached-cflinuxfs3-v4.4.9.zip           cflinuxfs3
dotnet_core_buildpack          14         true      false    dotnet-core_buildpack-cached-cflinuxfs3-v2.3.7.zip   cflinuxfs3
binary_buildpack               15         true      false    binary_buildpack-cached-cflinuxfs3-v1.0.36.zip       cflinuxfs3
binary_buildpack               16         true      false    binary_buildpack-cached-windows-v1.0.36.zip          windows
nginx_buildpack                17         true      false    nginx_buildpack-cached-cflinuxfs3-v1.1.6.zip         cflinuxfs3
r_buildpack                    18         true      false    r_buildpack-cached-cflinuxfs3-v1.1.2.zip             cflinuxfs3

Documents/workspace-spring-tool-suite-4-4.4.0.RELEASE/kaizen-doozer  master ✗                                                                                                                     5d ⚑ ✚ ◒  
▶ cf curl /v3/droplets/746491bf-afb9-4533-b3c3-91c4f5564e9d
{
   "guid": "746491bf-afb9-4533-b3c3-91c4f5564e9d",
   "state": "STAGED",
   "error": null,
   "lifecycle": {
      "type": "buildpack",
      "data": {}
   },
   "checksum": {
      "type": "sha256",
      "value": "cb351a2300dd0744d95c07732b24c31d7c1252022eb8a63bc71e446ade8f54d4"
   },
   "buildpacks": [
      {
         "name": "java_buildpack_offline",
         "detect_output": "",
         "buildpack_name": "",
         "version": null
      }
   ],
   "stack": "cflinuxfs3",
   "image": null,
   "execution_metadata": "",
   "process_types": {
      "task": "JAVA_OPTS=\"-agentpath:$PWD/.java-buildpack/open_jdk_jre/bin/jvmkill-1.16.0_RELEASE=printHeapHistogram=1 -Djava.io.tmpdir=$TMPDIR -XX:ActiveProcessorCount=$(nproc) -Djava.ext.dirs=$PWD/.java-buildpack/container_security_provider:$PWD/.java-buildpack/open_jdk_jre/lib/ext -Djava.security.properties=$PWD/.java-buildpack/java_security/java.security $JAVA_OPTS\" && CALCULATED_MEMORY=$($PWD/.java-buildpack/open_jdk_jre/bin/java-buildpack-memory-calculator-3.13.0_RELEASE -totMemory=$MEMORY_LIMIT -loadedClasses=12485 -poolType=metaspace -stackThreads=250 -vmOptions=\"$JAVA_OPTS\") && echo JVM Memory Configuration: $CALCULATED_MEMORY && JAVA_OPTS=\"$JAVA_OPTS $CALCULATED_MEMORY\" && MALLOC_ARENA_MAX=2 SERVER_PORT=$PORT eval exec $PWD/.java-buildpack/open_jdk_jre/bin/java $JAVA_OPTS -cp $PWD/. org.springframework.boot.loader.JarLauncher",
      "web": "JAVA_OPTS=\"-agentpath:$PWD/.java-buildpack/open_jdk_jre/bin/jvmkill-1.16.0_RELEASE=printHeapHistogram=1 -Djava.io.tmpdir=$TMPDIR -XX:ActiveProcessorCount=$(nproc) -Djava.ext.dirs=$PWD/.java-buildpack/container_security_provider:$PWD/.java-buildpack/open_jdk_jre/lib/ext -Djava.security.properties=$PWD/.java-buildpack/java_security/java.security $JAVA_OPTS\" && CALCULATED_MEMORY=$($PWD/.java-buildpack/open_jdk_jre/bin/java-buildpack-memory-calculator-3.13.0_RELEASE -totMemory=$MEMORY_LIMIT -loadedClasses=12485 -poolType=metaspace -stackThreads=250 -vmOptions=\"$JAVA_OPTS\") && echo JVM Memory Configuration: $CALCULATED_MEMORY && JAVA_OPTS=\"$JAVA_OPTS $CALCULATED_MEMORY\" && MALLOC_ARENA_MAX=2 SERVER_PORT=$PORT eval exec $PWD/.java-buildpack/open_jdk_jre/bin/java $JAVA_OPTS -cp $PWD/. org.springframework.boot.loader.JarLauncher"
   },
   "created_at": "2020-04-20T21:48:31Z",
   "updated_at": "2020-04-20T21:48:36Z",
   "relationships": {
      "app": {
         "data": {
            "guid": "7bf1e72d-ec90-4ec5-99e3-21f94535fc73"
         }
      }
   },
   "links": {
      "self": {
         "href": "https://api.sys.test.region.iaas.foundation.io/v3/droplets/746491bf-afb9-4533-b3c3-91c4f5564e9d"
      },
      "app": {
         "href": "https://api.sys.test.region.iaas.foundation.io/v3/apps/7bf1e72d-ec90-4ec5-99e3-21f94535fc73"
      },
      "assign_current_droplet": {
         "href": "https://api.sys.test.region.iaas.foundation.io/v3/apps/7bf1e72d-ec90-4ec5-99e3-21f94535fc73/relationships/current_droplet",
         "method": "PATCH"
      },
      "package": {
         "href": "https://api.sys.test.region.iaas.foundation.io/v3/packages/03e68633-6966-45d7-819e-367dd20c085d"
      }
   },
   "metadata": {
      "labels": {},
      "annotations": {}
   }
}

[stwomack]

Is this something that can be added?

[nebhale]

@xyloman There's some bit of difference between the two outputs that you've shown here; the .NET application allowed detection to happen (notice the value for detect_output) while the Java application had the buildpack explicitly declared (-b java_buildpack or similar). This has been a complaint since at least 2013; that buildpacks with explicit declaration don't get a bunch of really useful information from detect time. A curl of a detected Java application has quite a bit more information:

➜  ~/d/s/c/j/java-main-application main cf curl /v3/droplets/ffc5aec5-163c-492a-a19c-f03f0d7db1ab | jq .buildpacks
[
  {
    "name": "java_buildpack",
    "detect_output": "client-certificate-mapper=1.11.0_RELEASE container-security-provider=1.18.0_RELEASE java-buildpack=v4.31.1-offline-https://github.com/cloudfoundry/java-buildpack.git#03e8eec java-main java-opts java-security jvmkill-agent=1.16.0_RELEASE open-j...",
    "buildpack_name": "client-certificate-mapper=1.11.0_RELEASE container-security-provider=1.18.0_RELEASE java-buildpack=v4.31.1-offline-https://github.com/cloudfoundry/java-buildpack.git#03e8eec java-main java-opts java-security jvmkill-agent=1.16.0_RELEASE open-j...",
    "version": null
  }
]

It does not have the version field populated though, so I'll investigate what it takes to populate that.

[xyloman]

Very insightful thank you for sharing. To provide context the original goal of this request was to stream through all of the running / active droplets and compare the buildpacks version with the current version available. Based upon this information identify applications that might require restaging to remediate a CVE or identify those that might be vulnerable. Therefore, is there a better approach that could be taken or possibly one in the future based upon build service?

[nebhale]

The build service is definitely what you want to target for this. We expose so much more metadata there, and in a much more structured way. For example, not only will we tell you what JRE version was installed, we'll tell you its license, and a hash to ensure it hasn't been tampered with. Not only will we tell you what buildpack dependencies were contributed, we'll tell you what application dependencies were included.

My investigation has indicated that this is likely a simple addition to the packaged buildpacks and if it is, I'll add it. But at this point I'm loathe to do anything to the CF buildpacks that might destabilize them with Cloud Native Buildpacks so close.

[stwomack]

@nebhale Great information. Have you had time to think this through and make a decision?

[nebhale]

@xyloman and @stwomack finishing this up right now and wondering what you'd like to see version number-wise. We've always had relatively complex version number for people. Humans see things like

Java Buildpack bd9f4d6 | https://github.com/cloudfoundry/java-buildpack.git#bd9f4d6
Java Buildpack v4.31.1 (offline) | https://github.com/cloudfoundry/java-buildpack.git#03e8eec

and the machine-readable equivalents look like

bd9f4d6-https://github.com/cloudfoundry/java-buildpack.git#bd9f4d6
v4.31.1-offline-https://github.com/cloudfoundry/java-buildpack.git#03e8eec

Purely for consistency's sake, I'm inclined to use the same machine-readable string as we always have, but before I committed to that, I wanted to get your opinions.

[dashaun]

+1 to the machine-readable string

[nebhale]

I guess the question was what do you want that machine readable string to be? Options include

• v4.31.1-offline-https://github.com/cloudfoundry/java-buildpack.git#03e8eec
• v4.31.1-offline
• v4.31.1
• other

[xyloman]

I am wondering if this matching what is returned by the buildpack api would make the most sense?

▶ cf curl /v3/buildpacks
{
   "pagination": {
      "total_results": 18,
      "total_pages": 1,
      "first": {
         "href": "https://api.sys.test.region.iaas.foundation.io/v3/buildpacks?page=1&per_page=50"
      },
      "last": {
         "href": "https://api.sys.test.region.iaas.foundation.io/v3/buildpacks?page=1&per_page=50"
      },
      "next": null,
      "previous": null
   },
   "resources": [
...
      {
         "guid": "886a20fe-9e9b-4ea5-9439-79bf22e936a6",
         "created_at": "2018-10-15T22:10:36Z",
         "updated_at": "2020-06-25T20:13:15Z",
         "name": "java_buildpack_offline",
         "stack": "cflinuxfs3",
         "state": "READY",
         "filename": "java-buildpack-offline-cflinuxfs3-v4.31.zip",
         "position": 8,
         "enabled": true,
         "locked": false,
         "metadata": {
            "labels": {},
            "annotations": {}
         },
         "links": {
            "self": {
               "href": "https://api.sys.test.region.iaas.foundation.io/v3/buildpacks/886a20fe-9e9b-4ea5-9439-79bf22e936a6"
            },
            "upload": {
               "href": "https://api.sys.test.region.iaas.foundation.io/v3/buildpacks/886a20fe-9e9b-4ea5-9439-79bf22e936a6/upload",
               "method": "POST"
            }
         }
      },

...

     
   ]
}

So we would need enough information returned by the version that can be used to compare with the latest buildpack that exist on the platform. This would then allow us to identify applications that require updating to the latest buildpack. Given this would not be something that would be necessary when using build service supposedly or maybe this would still work in that world not sure.