Unverified Commit 92a7360d authored by azagrebin's avatar azagrebin Committed by GitHub
Browse files

Frocksdb release guide and helping scripts (#3)

parent d2156249
# Rocksdb Change Log
# FRocksdb Change Log
# 5.17.2-artisans (07/02/2019)
* [Flink TTL] compaction filter for background cleanup of state with time-to-live
* [Flink ListState] Separator-free merge operator
# 5.17.2 (10/24/2018)
### Bug Fixes
......
......@@ -6,6 +6,8 @@
#-----------------------------------------------
FROCKSDB_VERSION ?= 1.0
BASH_EXISTS := $(shell which bash)
SHELL := $(shell which bash)
......@@ -1783,17 +1785,72 @@ rocksdbjavastatic: $(java_static_all_libobjects)
strip $(STRIPFLAGS) $(ROCKSDBJNILIB); \
fi
cd java;jar -cf target/$(ROCKSDB_JAR) HISTORY*.md
jar -uf java/target/$(ROCKSDB_JAR) HISTORY*.md
cd java/target;jar -uf $(ROCKSDB_JAR) $(ROCKSDBJNILIB)
cd java/target/classes;jar -uf ../$(ROCKSDB_JAR) org/rocksdb/*.class org/rocksdb/util/*.class
cd java/target/apidocs;jar -cf ../$(ROCKSDB_JAVADOCS_JAR) *
cd java/src/main/java;jar -cf ../../../target/$(ROCKSDB_SOURCES_JAR) org
mkdir -p java/target/META-INF
cp LICENSE.Apache java/target/META-INF/LICENSE
cd java/target;jar -uf $(ROCKSDB_JAR) META-INF/LICENSE
rocksdbjavastaticrelease: rocksdbjavastatic
cd java/crossbuild && vagrant destroy -f && vagrant up linux32 && vagrant halt linux32 && vagrant up linux64 && vagrant halt linux64
cd java;jar -cf target/$(ROCKSDB_JAR_ALL) HISTORY*.md
cd java/target;jar -uf $(ROCKSDB_JAR_ALL) librocksdbjni-*.so librocksdbjni-*.jnilib
jar -uf java/target/$(ROCKSDB_JAR_ALL) HISTORY*.md
cd java/target;jar -uf $(ROCKSDB_JAR_ALL) librocksdbjni-*.so librocksdbjni-*.jnilib librocksdbjni-win64.dll
cd java/target/classes;jar -uf ../$(ROCKSDB_JAR_ALL) org/rocksdb/*.class org/rocksdb/util/*.class
frocksdbjavastaticrelease: rocksdbjavastaticrelease
# update license
mkdir -p java/target/META-INF
cp LICENSE.Apache java/target/META-INF/LICENSE
cd java/target;jar -uf $(ROCKSDB_JAR_ALL) META-INF/LICENSE
# platform jars
$(eval JAR_PREF=rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH))
$(eval JAR_DOCS=$(JAR_PREF)-javadoc.jar)
$(eval JAR_SOURCES=$(JAR_PREF)-sources.jar)
$(eval OSX_JAR=$(JAR_PREF)-osx.jar)
$(eval WIN_JAR=$(JAR_PREF)-win64.jar)
$(eval LINUX32_JAR=$(JAR_PREF)-linux32.jar)
$(eval LINUX64_JAR=$(JAR_PREF)-linux64.jar)
# update windows jar
cd java/target;cp rocksdbjni_classes.jar $(WIN_JAR)
cd java;jar -uf target/$(WIN_JAR) HISTORY*.md
jar -uf java/target/$(WIN_JAR) HISTORY*.md
cd java/target;jar -uf $(WIN_JAR) librocksdbjni-win64.dll
cd java/target;jar -uf $(WIN_JAR) META-INF/LICENSE
# update linux 64 jar with ppc64 lib
cd java/target;jar -uf $(LINUX64_JAR) librocksdbjni-linux-ppc64le.so
cd java/target;jar -uf $(JAR_DOCS) META-INF/LICENSE
cd java/target;jar -uf $(JAR_SOURCES) META-INF/LICENSE
# prepare frocksdb release
cd java/target;mkdir -p frocksdb-release
$(eval FJAR_PREF=frocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-artisans-$(FROCKSDB_VERSION))
$(eval FJAR=$(FJAR_PREF).jar)
$(eval FJAR_DOCS=$(FJAR_PREF)-javadoc.jar)
$(eval FJAR_SOURCES=$(FJAR_PREF)-sources.jar)
$(eval OSX_FJAR=$(FJAR_PREF)-osx.jar)
$(eval WIN_FJAR=$(FJAR_PREF)-win64.jar)
$(eval LINUX32_FJAR=$(FJAR_PREF)-linux32.jar)
$(eval LINUX64_FJAR=$(FJAR_PREF)-linux64.jar)
cd java/target;cp $(ROCKSDB_JAR_ALL) frocksdb-release/$(FJAR)
cd java/target;cp $(JAR_DOCS) frocksdb-release/$(FJAR_DOCS)
cd java/target;cp $(JAR_SOURCES) frocksdb-release/$(FJAR_SOURCES)
cd java/target;cp $(OSX_JAR) frocksdb-release/$(OSX_FJAR)
cd java/target;cp $(WIN_JAR) frocksdb-release/$(WIN_FJAR)
cd java/target;cp $(LINUX32_JAR) frocksdb-release/$(LINUX32_FJAR)
cd java/target;cp $(LINUX64_JAR) frocksdb-release/$(LINUX64_FJAR)
cd java;cp rocksjni.pom target/frocksdb-release/$(FJAR_PREF).pom
rocksdbjavastaticreleasedocker: rocksdbjavastatic rocksdbjavastaticdockerx86 rocksdbjavastaticdockerx86_64
cd java;jar -cf target/$(ROCKSDB_JAR_ALL) HISTORY*.md
cd java/target;jar -uf $(ROCKSDB_JAR_ALL) librocksdbjni-*.so librocksdbjni-*.jnilib
......
## Build in Windows
Use Windows 64 bit machine (e.g. base AWS Windows instance: 4 cores, 16GB RAM, 40GB storage for build).
Install:
* git
* java 8
* maven
* Visual Studio Community 15 (2017)
With [chocolatey](https://chocolatey.org/install):
choco install git.install jdk8 maven visualstudio2017community
Optionally:
choco install intellijidea-community vscode
Open git bash and run commands:
git clone git@github.com:dataArtisans/frocksdb.git
cd frocksdb
git checkout FRocksDB-5.17.2 # release branch
java/crossbuild/build-win.bat
The result native library is `build\java\Release\rocksdbjni-shared.dll`.
The result windows jar is `build\java\rocksdbjni_classes.jar`.
There is also a how-to in CMakeLists.txt.
## Build in PPC64LE
Use Ubuntu 16.04 (e.g. AWS instance 4 cores, 16GB RAM, 40GB storage for build).
Install git if not installed. If docker is installed, it might need to be removed.
Setup ppc64le docker machine ([source](https://developer.ibm.com/linuxonpower/2017/06/08/build-test-ppc64le-docker-images-intel/)):
wget http://ftp.unicamp.br/pub/ppc64el/boot2docker/install.sh && chmod +x ./install.sh && ./install.sh -s
docker-machine create -d qemu \
--qemu-boot2docker-url=/home/ubuntu/.docker/machine/boot2docker.iso \
--qemu-memory 8192 \
--qemu-cache-mode none \
--qemu-arch ppc64le \
vm-ppc64le
Regenerate certs as suggested if it did not work at once.
Prepare docker machine to run rocksdbjni docker image for ppc64le build:
eval $(docker-machine env vm-ppc64le)
git clone git@github.com:dataArtisans/frocksdb.git
cd frocksdb
git checkout FRocksDB-5.17.2 # release branch
docker-machine ssh vm-ppc64le mkdir -p `pwd`
docker-machine scp -r . vm-ppc64le:`pwd`
Build frocksdb:
make rocksdbjavastaticdockerppc64le
docker-machine scp vm-ppc64le:`pwd`/java/target/librocksdbjni-linux-ppc64le.so java/target/.
The result native library is in `java/target/librocksdbjni-linux-ppc64le.so`.
## Final crossbuild in Mac OSX
Read how to Build cross jar for Mac OSX and linux as described in java/RELEASE.md but do not run it yet.
Run commands:
make jclean clean
mkdir -p java/target
cp <path-to-windows-dll>/rocksdbjni-shared.dll java/target/librocksdbjni-win64.dll
cp <path-to-windows-jar>/rocksdbjni_classes.jar java/target/rocksdbjni_classes.jar
cp <path-to-ppc64le-lib-so>/librocksdbjni-linux-ppc64le.so java/target/librocksdbjni-linux-ppc64le.so
FROCKSDB_VERSION=1.0 make frocksdbjavastaticrelease
## Push to maven central
Edit the `frocksdbjni-<release version>.pom` file and replace
`<version>-</version>`
with
`<version><release version></version>`
where the `<release version>` is e.g. `5.17.2-artisans-1.0`.
Run:
```bash
VERSION=<release version> \
USER=<sonatype user> \
PASSWORD=<sonatype password> \
KEYNAME=<key name> \
PASSPHRASE=<passphrase> \
java/publish-frocksdbjni.sh
```
Go to the staging repositories on Sonatype:
https://oss.sonatype.org/#stagingRepositories
Select the open staging repository and click on "Close".
Test the files in the staging repository
which will look something like this `https://oss.sonatype.org/content/repositories/comdata-artisans-1020`.
Press the "Release" button (WARNING: this can not be undone).
......@@ -26,6 +26,6 @@ export JAVA_HOME=/usr/lib/jvm/java-1.7.0
# build rocksdb
cd /rocksdb
scl enable devtoolset-2 'make jclean clean'
scl enable devtoolset-2 'PORTABLE=1 make -j8 rocksdbjavastatic'
scl enable devtoolset-2 'PORTABLE=1 make rocksdbjavastatic'
cp /rocksdb/java/target/librocksdbjni-* /rocksdb-build
cp /rocksdb/java/target/rocksdbjni-* /rocksdb-build
......@@ -7,7 +7,7 @@ sudo apt-get -y install git make gcc g++ libgflags-dev libsnappy-dev zlib1g-dev
export JAVA_HOME=$(echo /usr/lib/jvm/java-7-openjdk*)
cd /rocksdb
make jclean clean
make -j 4 rocksdbjavastatic
make rocksdbjavastatic
cp /rocksdb/java/target/librocksdbjni-* /rocksdb-build
cp /rocksdb/java/target/rocksdbjni-* /rocksdb-build
sudo shutdown -h now
......
:: install git, java 8, maven, visual studio community 15 (2017)
set MSBUILD=C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\MSBuild\15.0\Bin\MSBuild.exe
if exist build rd /s /q build
if exist librocksdbjni-win64.dll del librocksdbjni-win64.dll
mkdir build && cd build
cmake -G "Visual Studio 15 Win64" -DWITH_JNI=1 ..
"%MSBUILD%" rocksdb.sln /p:Configuration=Release /m
cd ..
copy build\Release\rocksdb-shared.dll librocksdbjni-win64.dll
echo Result is in librocksdbjni-win64.dll
\ No newline at end of file
......@@ -9,10 +9,10 @@ cd /rocksdb-local
# Use scl devtoolset if available (i.e. CentOS <7)
if hash scl 2>/dev/null; then
scl enable devtoolset-2 'make jclean clean'
scl enable devtoolset-2 'PORTABLE=1 make -j8 rocksdbjavastatic'
scl enable devtoolset-2 'PORTABLE=1 make rocksdbjavastatic'
else
make jclean clean
PORTABLE=1 make -j8 rocksdbjavastatic
PORTABLE=1 make rocksdbjavastatic
fi
cp java/target/librocksdbjni-linux*.so java/target/rocksdbjni-*-linux*.jar /rocksdb-host/java/target
......
<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0
http://maven.apache.org/xsd/settings-1.0.0.xsd">
<servers>
<server>
<id>sonatype-nexus-staging</id>
<username>${sonatype_user}</username>
<password>${sonatype_pw}</password>
</server>
</servers>
</settings>
#!/usr/bin/env bash
################################################################################
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
################################################################################
# fail on errors
set -e
function deploy() {
FILE=$1
CLASSIFIER=$2
echo "Deploying file=${FILE} with classifier=${CLASSIFIER} to sonatype"
sonatype_user=${USER} sonatype_pw=${PASSWORD} mvn gpg:sign-and-deploy-file \
--settings java/deploysettings.xml \
-Durl=https://oss.sonatype.org/service/local/staging/deploy/maven2/ \
-DrepositoryId=sonatype-nexus-staging \
-DpomFile=java/frocksdbjni-${VERSION}.pom \
-Dfile=$FILE \
-Dclassifier=$CLASSIFIER \
-Dgpg.keyname=${KEYNAME} \
-Dgpg.passphrase=${PASSPHRASE}
}
PREFIX=java/target/frocksdb-release/frocksdbjni-${VERSION}
deploy ${PREFIX}-sources.jar sources
deploy ${PREFIX}-javadoc.jar javadoc
deploy ${PREFIX}.jar
......@@ -4,14 +4,14 @@
xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<modelVersion>4.0.0</modelVersion>
<name>RocksDB JNI</name>
<url>http://rocksdb.org/</url>
<groupId>org.rocksdb</groupId>
<artifactId>rocksdbjni</artifactId>
<name>FRocksDB JNI</name>
<url>https://github.com/dataArtisans/frocksdb</url>
<groupId>com.data-artisans</groupId>
<artifactId>frocksdbjni</artifactId>
<!-- Version will be automatically replaced -->
<version>-</version>
<description>RocksDB fat jar that contains .so files for linux32 and linux64, jnilib files
for Mac OSX, and a .dll for Windows x64.
<description>RocksDB fat jar with modifications specific for Apache Flink
that contains .so files for linux32 and linux64, jnilib files for Mac OSX, and a .dll for Windows x64.
</description>
<licenses>
<license>
......@@ -19,11 +19,6 @@
<url>http://www.apache.org/licenses/LICENSE-2.0.html</url>
<distribution>repo</distribution>
</license>
<license>
<name>GNU General Public License, version 2</name>
<url>http://www.gnu.org/licenses/gpl-2.0.html</url>
<distribution>repo</distribution>
</license>
</licenses>
<scm>
<connection>scm:git:git://github.com/dropwizard/metrics.git</connection>
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment