WT-3021 fixes for java log example, raw mode in java, and raw mode in log cursors (#3150)

* log cursors performing search must internally get keys; raw mode must be turned off temporarily to accomplish that. Use this same technique to clean up how keys/values are set.

* In java, when a raw mode cursor is created, allow setting key/value only via byte array.

* Synchronize java log example with the C version, including changing LSN from long to int and adding a count check to the simple log walk. Fix an error with how LSN file/offset was saved for a later search. The 'copy' table can now use S formats, since there is a way to insert them as raw records obtained from the log.

4 files changed, 82 insertions, 77 deletions

diff --git a/examples/java/com/wiredtiger/examples/ex_log.java b/examples/java/com/wiredtiger/examples/ex_log.java
index 03eab6b38b1..233ad1361d8 100644
--- a/examples/java/com/wiredtiger/examples/ex_log.java
+++ b/examples/java/com/wiredtiger/examples/ex_log.java
@@ -35,7 +35,7 @@ import java.util.*;
 
 class Lsn {
     int file;
-    long offset;
+    int offset;
 }
 
 public class ex_log {
@@ -57,7 +57,7 @@ public class ex_log {
 
         conn = wiredtiger.open(home2, CONN_CONFIG);
         Session session = conn.open_session(null);
-        session.create(uri, "key_format=U,value_format=U");
+        session.create(uri, "key_format=S,value_format=S");
         return (session);
     }
 
@@ -72,17 +72,17 @@ public class ex_log {
 
         while ((ret = cursor.next()) == 0) {
             ret = curs_copy.next();
-            byte[] key = cursor.getKeyByteArray();
-            byte[] value = cursor.getValueByteArray();
-            byte[] key_copy = curs_copy.getKeyByteArray();
-            byte[] value_copy = curs_copy.getValueByteArray();
-            if (!Arrays.equals(key, key_copy) ||
-                !Arrays.equals(value, value_copy)) {
+            String key = cursor.getKeyString();
+            String value = cursor.getValueString();
+            String key_copy = curs_copy.getKeyString();
+            String value_copy = curs_copy.getValueString();
+            if (!key.equals(key_copy) ||
+                !value.equals(value_copy)) {
                 System.err.println(
-                    "Mismatched: key " + new String(key) +
-                    ", key_copy " + new String(key_copy) +
-                    ", value " + new String(value) +
-                    ", value_copy " + new String(value_copy));
+                    "Mismatched: key " + key + ", key_copy " + key_copy +
+                    ", value " + value + ", value_copy " + value_copy);
+                ret = cursor.close();
+                ret = curs_copy.close();
                 return (1);
             }
         }
@@ -121,7 +121,7 @@ public class ex_log {
      *	A simple walk of the log.
      */
     static int
-    simple_walk_log(Session session)
+    simple_walk_log(Session session, int count_min)
         throws WiredTigerException
     {
         Cursor cursor;
@@ -129,16 +129,18 @@ public class ex_log {
         byte[] logrec_key, logrec_value;
         long txnid;
         int fileid, opcount, optype, rectype;
-        int ret;
+        int count, ret;
 
         /*! [log cursor open] */
         cursor = session.open_cursor("log:", null, null);
         /*! [log cursor open] */
 
+        count = 0;
         while ((ret = cursor.next()) == 0) {
+            count++;
             /*! [log cursor get_key] */
             lsn.file = cursor.getKeyInt();
-            lsn.offset = cursor.getKeyLong();
+            lsn.offset = cursor.getKeyInt();
             opcount = cursor.getKeyInt();
             /*! [log cursor get_key] */
             /*! [log cursor get_value] */
@@ -156,6 +158,11 @@ public class ex_log {
         if (ret == wiredtiger.WT_NOTFOUND)
             ret = 0;
         ret = cursor.close();
+	if (count < count_min) {
+            System.err.println("Expected minimum " + count_min +
+                               " records, found " + count);
+            return (1);
+        }
         return (ret);
     }
     /*! [log cursor walk] */
@@ -185,7 +192,7 @@ public class ex_log {
         lsnsave = new Lsn();
         while ((ret = cursor.next()) == 0) {
             lsn.file = cursor.getKeyInt();
-            lsn.offset = cursor.getKeyLong();
+            lsn.offset = cursor.getKeyInt();
             opcount = cursor.getKeyInt();
 
             /*
@@ -194,8 +201,10 @@ public class ex_log {
              * that LSN to the end (where the multi-step transaction
              * was performed).  Just choose the record that is MAX_KEYS.
              */
-            if (++i == MAX_KEYS)
-                lsnsave = lsn;
+            if (++i == MAX_KEYS) {
+                lsnsave.file = lsn.file;
+                lsnsave.offset = lsn.offset;
+            }
             txnid = cursor.getValueLong();
             rectype = cursor.getValueInt();
             optype = cursor.getValueInt();
@@ -217,10 +226,10 @@ public class ex_log {
 
             /*
              * If the operation is a put, replay it here on the backup
-             * connection.  Note, we cheat by looking only for fileid 1
-             * in this example.  The metadata is fileid 0.
+             * connection.  Note, we cheat by looking at the fileid.
+             * The metadata is fileid 0, skip its records.
              */
-            if (fileid == 1 && rectype == wiredtiger.WT_LOGREC_COMMIT &&
+            if (fileid != 0 && rectype == wiredtiger.WT_LOGREC_COMMIT &&
                 optype == wiredtiger.WT_LOGOP_ROW_PUT) {
                 if (!in_txn) {
                     ret = session2.begin_transaction(null);
@@ -238,15 +247,20 @@ public class ex_log {
         /*
          * Compare the tables after replay.  They should be identical.
          */
-        if (compare_tables(session, session2) != 0)
-            System.out.println("compare failed");
+        if (compare_tables(session, session2) != 0) {
+            cursor.close();
+            session2.close(null);
+            wt_conn2.close(null);
+            return (ret);
+        }
         ret = session2.close(null);
         ret = wt_conn2.close(null);
 
         ret = cursor.reset();
         /*! [log cursor set_key] */
         cursor.putKeyInt(lsnsave.file);
-        cursor.putKeyLong(lsnsave.offset);
+        cursor.putKeyInt(lsnsave.offset);
+        cursor.putKeyInt(0);
         /*! [log cursor set_key] */
         /*! [log cursor search] */
         ret = cursor.search();
@@ -256,9 +270,9 @@ public class ex_log {
          * Walk all records starting with this key.
          */
         first = true;
-        while (ret == 0) {  /*TODO: not quite right*/
+        while (ret == 0) {
             lsn.file = cursor.getKeyInt();
-            lsn.offset = cursor.getKeyLong();
+            lsn.offset = cursor.getKeyInt();
             opcount = cursor.getKeyInt();
             if (first) {
                 first = false;
@@ -293,8 +307,9 @@ public class ex_log {
         Connection wt_conn;
         Cursor cursor;
         Session session;
-        int i, record_count, ret;
+        int count_min, i, record_count, ret;
 
+        count_min = 0;
         try {
             String command = "/bin/rm -rf " + home1 + " " + home2;
             Process proc = Runtime.getRuntime().exec(command);
@@ -317,6 +332,7 @@ public class ex_log {
 
         session = wt_conn.open_session(null);
         ret = session.create(uri, "key_format=S,value_format=S");
+        count_min++;
 
         cursor = session.open_cursor(uri, null, null);
         /*
@@ -328,6 +344,7 @@ public class ex_log {
             cursor.putKeyString(k);
             cursor.putValueString(v);
             ret = cursor.insert();
+            count_min++;
         }
         ret = session.begin_transaction(null);
         /*
@@ -341,10 +358,12 @@ public class ex_log {
             ret = cursor.insert();
         }
         ret = session.commit_transaction(null);
+        count_min++;
         ret = cursor.close();
 
         /*! [log cursor printf] */
         ret = session.log_printf("Wrote " + record_count + " records");
+        count_min++;
         /*! [log cursor printf] */
 
         session.close(null);
@@ -360,7 +379,7 @@ public class ex_log {
         }
 
         session = wt_conn.open_session(null);
-        ret = simple_walk_log(session);
+        ret = simple_walk_log(session, count_min);
         ret = walk_log(session);
         ret = session.close(null);
         ret = wt_conn.close(null);
diff --git a/lang/java/src/com/wiredtiger/db/PackOutputStream.java b/lang/java/src/com/wiredtiger/db/PackOutputStream.java
index 46b3aef0974..0ccfdf6c413 100644
--- a/lang/java/src/com/wiredtiger/db/PackOutputStream.java
+++ b/lang/java/src/com/wiredtiger/db/PackOutputStream.java
@@ -42,6 +42,7 @@ public class PackOutputStream {
     protected PackFormatInputStream format;
     protected ByteArrayOutputStream packed;
     protected byte[] intBuf;
+    protected boolean isRaw;
 
     /**
      * Constructor.
@@ -49,10 +50,11 @@ public class PackOutputStream {
      * \param format A String that contains the WiredTiger format that
      *               defines the layout of this packed value.
      */
-    public PackOutputStream(String format) {
-        this.format = new PackFormatInputStream(format, false);
-        intBuf = new byte[MAX_INT_BYTES];
-        packed = new ByteArrayOutputStream(100);
+    public PackOutputStream(String format, boolean isRaw) {
+        this.format = new PackFormatInputStream(format, isRaw);
+        this.intBuf = new byte[MAX_INT_BYTES];
+        this.packed = new ByteArrayOutputStream(100);
+        this.isRaw = isRaw;
     }
 
     /**
@@ -111,7 +113,9 @@ public class PackOutputStream {
     throws WiredTigerPackingException {
         int padBytes = 0;
 
-        format.checkType('U', false);
+        if (!isRaw) {
+            format.checkType('U', false);
+        }
         boolean havesize = format.hasLength();
         char type = format.getType();
         if (havesize) {
diff --git a/lang/java/wiredtiger.i b/lang/java/wiredtiger.i
index d6e0660dc2c..b7515440040 100644
--- a/lang/java/wiredtiger.i
+++ b/lang/java/wiredtiger.i
@@ -609,8 +609,8 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler};
    swigCPtr = cPtr;
    keyFormat = getKey_format();
    valueFormat = getValue_format();
-   keyPacker = new PackOutputStream(keyFormat);
-   valuePacker = new PackOutputStream(valueFormat);
+   keyPacker = new PackOutputStream(keyFormat, _java_raw());
+   valuePacker = new PackOutputStream(valueFormat, _java_raw());
    wiredtigerJNI.AsyncOp__java_init(swigCPtr, this, this);
  }
 
@@ -1232,8 +1232,8 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler};
    swigCPtr = cPtr;
    keyFormat = getKey_format();
    valueFormat = getValue_format();
-   keyPacker = new PackOutputStream(keyFormat);
-   valuePacker = new PackOutputStream(valueFormat);
+   keyPacker = new PackOutputStream(keyFormat, _java_raw());
+   valuePacker = new PackOutputStream(valueFormat, _java_raw());
    wiredtigerJNI.Cursor__java_init(swigCPtr, this, this);
  }
 
diff --git a/src/cursor/cur_log.c b/src/cursor/cur_log.c
index 21a7f674c68..3ee6554b3c0 100644
--- a/src/cursor/cur_log.c
+++ b/src/cursor/cur_log.c
@@ -150,18 +150,22 @@ static int
 __curlog_kv(WT_SESSION_IMPL *session, WT_CURSOR *cursor)
 {
 	WT_CURSOR_LOG *cl;
-	WT_ITEM item;
-	uint32_t fileid, key_count, opsize, optype;
+	WT_DECL_RET;
+	uint32_t fileid, key_count, opsize, optype, raw;
 
 	cl = (WT_CURSOR_LOG *)cursor;
+	/* Temporarily turn off raw so we can do direct cursor operations. */
+	raw = F_MASK(cursor, WT_CURSTD_RAW);
+	F_CLR(cursor, WT_CURSTD_RAW);
+
 	/*
 	 * If it is a commit and we have stepped over the header, peek to get
 	 * the size and optype and read out any key/value from this operation.
 	 */
 	if ((key_count = cl->step_count++) > 0) {
-		WT_RET(__wt_logop_read(session,
+		WT_ERR(__wt_logop_read(session,
 		    &cl->stepp, cl->stepp_end, &optype, &opsize));
-		WT_RET(__curlog_op_read(session, cl, optype, opsize, &fileid));
+		WT_ERR(__curlog_op_read(session, cl, optype, opsize, &fileid));
 		/* Position on the beginning of the next record part. */
 		cl->stepp += opsize;
 	} else {
@@ -181,39 +185,14 @@ __curlog_kv(WT_SESSION_IMPL *session, WT_CURSOR *cursor)
 	 * The log cursor sets the LSN and step count as the cursor key and
 	 * and log record related data in the value.  The data in the value
 	 * contains any operation key/value that was in the log record.
-	 * For the special case that the caller needs the result in raw form,
-	 * we create packed versions of the key/value.
 	 */
-	if (FLD_ISSET(cursor->flags, WT_CURSTD_RAW)) {
-		memset(&item, 0, sizeof(item));
-		WT_RET(wiredtiger_struct_size((WT_SESSION *)session,
-		    &item.size, WT_LOGC_KEY_FORMAT, cl->cur_lsn->l.file,
-		    cl->cur_lsn->l.offset, key_count));
-		WT_RET(__wt_realloc(session, NULL, item.size, &cl->packed_key));
-		item.data = cl->packed_key;
-		WT_RET(wiredtiger_struct_pack((WT_SESSION *)session,
-		    cl->packed_key, item.size, WT_LOGC_KEY_FORMAT,
-		    cl->cur_lsn->l.file, cl->cur_lsn->l.offset, key_count));
-		__wt_cursor_set_key(cursor, &item);
-
-		WT_RET(wiredtiger_struct_size((WT_SESSION *)session,
-		    &item.size, WT_LOGC_VALUE_FORMAT, cl->txnid, cl->rectype,
-		    optype, fileid, cl->opkey, cl->opvalue));
-		WT_RET(__wt_realloc(session, NULL, item.size,
-		    &cl->packed_value));
-		item.data = cl->packed_value;
-		WT_RET(wiredtiger_struct_pack((WT_SESSION *)session,
-		    cl->packed_value, item.size, WT_LOGC_VALUE_FORMAT,
-		    cl->txnid, cl->rectype, optype, fileid, cl->opkey,
-		    cl->opvalue));
-		__wt_cursor_set_value(cursor, &item);
-	} else {
-		__wt_cursor_set_key(cursor, cl->cur_lsn->l.file,
-		    cl->cur_lsn->l.offset, key_count);
-		__wt_cursor_set_value(cursor, cl->txnid, cl->rectype, optype,
-		    fileid, cl->opkey, cl->opvalue);
-	}
-	return (0);
+	__wt_cursor_set_key(cursor, cl->cur_lsn->l.file, cl->cur_lsn->l.offset,
+	    key_count);
+	__wt_cursor_set_value(cursor, cl->txnid, cl->rectype, optype, fileid,
+	    cl->opkey, cl->opvalue);
+
+err:	F_SET(cursor, raw);
+	return (ret);
 }
 
 /*
@@ -264,17 +243,19 @@ __curlog_search(WT_CURSOR *cursor)
 	WT_DECL_RET;
 	WT_LSN key;
 	WT_SESSION_IMPL *session;
-	uint32_t counter, key_file, key_offset;
+	uint32_t counter, key_file, key_offset, raw;
 
 	cl = (WT_CURSOR_LOG *)cursor;
+	/* Temporarily turn off raw so we can do direct cursor operations. */
+	raw = F_MASK(cursor, WT_CURSTD_RAW);
+	F_CLR(cursor, WT_CURSTD_RAW);
 
 	CURSOR_API_CALL(cursor, session, search, NULL);
 
 	/*
 	 * !!! We are ignoring the counter and only searching based on the LSN.
 	 */
-	WT_ERR(__wt_cursor_get_key((WT_CURSOR *)cl,
-	    &key_file, &key_offset, &counter));
+	WT_ERR(__wt_cursor_get_key(cursor, &key_file, &key_offset, &counter));
 	WT_SET_LSN(&key, key_file, key_offset);
 	ret = __wt_log_scan(session, &key, WT_LOGSCAN_ONE,
 	    __curlog_logrec, cl);
@@ -285,7 +266,8 @@ __curlog_search(WT_CURSOR *cursor)
 	WT_STAT_CONN_INCR(session, cursor_search);
 	WT_STAT_DATA_INCR(session, cursor_search);
 
-err:	API_END_RET(session, ret);
+err:	F_SET(cursor, raw);
+	API_END_RET(session, ret);
 }
 
 /*