You can use Java Open Client DATASET-HANDLE and TABLE-HANDLE parameters to send/receive datasets and temp-tables to/from the application server, as shown in the following example code.

ABL code

dynOutDataSet.p 
DEFINE INPUT PARAMETER minCredit AS INTEGER.
DEFINE INPUT PARAMETER maxCredit AS INTEGER.

DEFINE OUTPUT PARAMETER DATASET-HANDLE dsetHandle.
dynInpOutDataSet.p 
DEFINE INPUT PARAMETER moreCredit AS INTEGER.

DEFINE INPUT-OUTPUT PARAMETER DATASET-HANDLE dsetHandle.

DEFINE TEMP-TABLE ttCust 
        FIELD CustNum LIKE Customer.Custnum 
        FIELD NAME LIKE Customer.NAME 
        FIELD Address LIKE Customer.Address 
        FIELD City LIKE Customer.City 
        FIELD CreditLimit LIKE Customer.CreditLimit .

DEFINE DATASET myDS FOR ttCust.
dynInpDataSet.p 
DEFINE INPUT PARAMETER DATASET-HANDLE dsetHandle.
dynOutTable.p 
DEFINE INPUT PARAMETER minCredit AS INTEGER.
DEFINE INPUT PARAMETER maxCredit AS INTEGER.
DEFINE OUTPUT PARAMETER TABLE-HANDLE ttHandle.

DEFINE TEMP-TABLE ttCust 
        FIELD CustNum LIKE Customer.Custnum 
        FIELD NAME LIKE Customer.NAME 
        FIELD Address LIKE Customer.Address 
        FIELD City LIKE Customer.City 
        FIELD CreditLimit LIKE Customer.CreditLimit .
      
DEFINE DATASET myDS FOR ttCust.
dynInpOutTable.p 
DEFINE INPUT PARAMETER moreCredit AS INTEGER.
DEFINE INPUT-OUTPUT PARAMETER TABLE-HANDLE ttHandle.
dynInpTable.p 
DEFINE INPUT PARAMETER TABLE-HANDLE ttHandle.

OpenAPI code

//Example code using DataSet-Handle and TempTable-Handle parameters on Java client

import java.util.List;
import com.progress.open4gl.*;
import com.progress.open4gl.javaproxy.*;


public class App {

    public static void main(String[] args) throws Exception {
    	
    	try
    	{
    		System.out.println("DataSet Example");
	        Connection myConn = new Connection("http://<host>/apsv", "", "", "");   
	        
	        Integer minCreditLimit = 1500;
	        Integer maxCreditLimit = 2000;
	        Integer moreCredit     = 5000;
	            	
	        OpenAppObject dynAO = new OpenAppObject(myConn,"");
	        
	        //////////////////////////////////////////////////////////////
	        /// Example code passing DataSet-Handle as output parameter
	
	        // Create the ParamArray
	        ParamArray parms = new ParamArray(3);
	
	        // Set up input parameters
	        parms.addInteger(0, minCreditLimit, ParamArrayMode.INPUT);
	        parms.addInteger(1, maxCreditLimit, ParamArrayMode.INPUT);
	
	        // Set up Out parameters.
	        // Okay to not specify metadata
	        parms.addDatasetHandle(2, null, ParamArrayMode.OUTPUT, null);
	        
	        // Run the procedure
	        System.out.println("Calling dynOutDataSet.p.");
	        dynAO.runProc("dynOutDataSet.p", parms);
        
	        ProDataGraph pdg = (ProDataGraph) parms.getOutputParameter(2);
	          
	        String tableName = pdg.getTableNames()[0];
	        List custList = pdg.getProDataObjects(tableName);
	        
	        System.out.println(tableName + " has " + custList.size() + " records."); 
	        
	        /////////////////////////////////////////////////////////////////
	        /// Example code passing DataSet-Handle as input-output parameter
	        
	        parms = new ParamArray(2);
	    	
	        // Set up input parameters
	        parms.addInteger(0, moreCredit, ParamArrayMode.INPUT);
	
	        // Set up Inp-Out parameters.
	        ProDataGraphMetaData dgmd = createDGMetaData();
	        parms.addDatasetHandle(1, pdg, ParamArrayMode.INPUT_OUTPUT, dgmd);
	        
	        System.out.println("Calling dynInpOutDataSet.p");
	        dynAO.runProc("dynInpOutDataSet.p", parms);
	        
	        pdg = (ProDataGraph) parms.getOutputParameter(1);
	        custList = pdg.getProDataObjects(pdg.getTableNames()[0]);
	        System.out.println(pdg.getTableNames()[0] + " from dynInpOutDataSet.p has " + custList.size() + " records."); 
	        
	        /////////////////////////////////////////////////////////////////
	        /// Example code passing DataSet-Handle as input parameter
	                
	        parms = new ParamArray(1);
	    	
	        parms.addDatasetHandle(0, pdg, ParamArrayMode.INPUT, dgmd);
	        
	        System.out.println("Calling dynInpDataSet.p");
	        dynAO.runProc("dynInpDataSet.p", parms);
	        System.out.println("Completed dynInpDataSet.p");
	        
	        doSomeTempTableWork(dynAO, pdg);
    	}
    	catch (Exception e)
    	{
    		System.out.println(e.getMessage());
    		e.printStackTrace();
    	}
        finally
    	{
    	}
    }
    
    
    public static ProDataGraphMetaData createDGMetaData() throws ProDataException
    {  	
    	ProDataGraphMetaData dgmd = null;
    	ProDataObjectMetaData custmd = null;
    	
    	try
    	{
    		dgmd = new ProDataGraphMetaData("myCustomerTable");	
	        
	        custmd = new ProDataObjectMetaData("ttCust", 5, false, 0, "", "", "");
	        custmd.setPrimeUniqueName("CustNum");
	        
	        custmd.setFieldMetaData(1, "CustNum", 0, Parameter.PRO_INTEGER, 0, 0);
	        custmd.setFieldMetaData(2, "Name", 0, Parameter.PRO_CHARACTER, 0, 0);
	        custmd.setFieldMetaData(3, "Address", 0, Parameter.PRO_CHARACTER, 0, 0);
	        custmd.setFieldMetaData(4, "City", 0, Parameter.PRO_CHARACTER, 0, 0);
	        custmd.setFieldMetaData(5, "CreditLimit", 0, Parameter.PRO_DECIMAL, 0, 0); 
	        
	        dgmd.addTable(custmd);
    	}
    	catch (Exception e)
    	{
    		System.out.println(e.getMessage());
    		e.printStackTrace();
    	}
    	finally
    	{
    	}
    	
        return dgmd;
    }
    
    
    public static void doSomeTempTableWork(OpenAppObject dynAO, ProDataGraph pdg) 
    					throws Exception
    {  	
    	ProDataGraph pdgOut;
        List custList;
    	
    	try
    	{	
    		System.out.println("\n\nTempTable Example");
    		Integer minCreditLimit = 1500;
	        Integer maxCreditLimit = 2000;
	        Integer moreCredit     = 5000;
            
            //////////////////////////////////////////////////////////////
            /// Example code passing Table-Handle as output parameter
            
    		// Set up input parameters
    		ParamArray parms = new ParamArray(3);
    		parms.addInteger(0, minCreditLimit, ParamArrayMode.INPUT);
    		parms.addInteger(1, maxCreditLimit, ParamArrayMode.INPUT);

    		// Set up output parameters
    		// Okay to not specify metadata
    		// For Java OpenClient, a temp-table is represented as a ProDataGrpah with a single table
    		parms.addTableHandle(2, pdg, ParamArrayMode.OUTPUT, null);

    		System.out.println("Calling dynOutTable.p");
    		dynAO.runProc("dynOutTable.p", parms);
    		
    		pdgOut = (ProDataGraph) parms.getOutputParameter(2);
    		custList = pdg.getProDataObjects(pdgOut.getTableNames()[0]);
    		System.out.println(pdgOut.getTableNames()[0] + " from dynOutTable.p has " + custList.size() + " records."); 	
    		
    		/////////////////////////////////////////////////////////////////
    		/// Example code passing Table-Handle as input-output parameter

    		// Run the procedure
    		parms = new ParamArray(2);
    		parms.addInteger(0, moreCredit, ParamArrayMode.INPUT);
    		
    		// Set up input-output parameters
    		// For Java OpenClient, a temp-table is represented as a ProDataGraph with a single table
    		ProDataGraphMetaData dgmd = createDGMetaData();
    		parms.addTableHandle(1, pdg, ParamArrayMode.INPUT_OUTPUT, dgmd);
    		System.out.println("Calling dynInpOutTable.p");
    		dynAO.runProc("dynInpOutTable.p", parms);
    		
    		pdgOut = (ProDataGraph) parms.getOutputParameter(1);
            custList = pdgOut.getProDataObjects(pdgOut.getTableNames()[0]);
            System.out.println(pdgOut.getTableNames()[0] + " from dynInpOutTable.p has " + custList.size() + " records.");       
    	

            /////////////////////////////////////////////////////////////////
            /// Example code passing Table-Handle as input parameter
            
            // Run the procedure
    		parms = new ParamArray(1);
    		
    		// Set up input parameters
    		// For Java OpenClient, a temp-table is represented as a ProDataGraph with a single table
    		parms.addTableHandle(0, pdg, ParamArrayMode.INPUT, dgmd);
    		System.out.println("Calling dynInpTable.p");
    		dynAO.runProc("dynInpTable.p", parms);
    		System.out.println("Completed dynInpTable.p");    
    	}
    		
    	catch (Exception e)
    	{
    		System.out.println(e.getMessage());
    		e.printStackTrace();
    	}
    	finally
    	{
    	}   	
    }
}