• Home
  • LLMs
  • Docker
  • Kubernetes
  • Java
  • All
  • About
Big Data | Pail (writing/reading Objects)
  1. References
  2. The project Structure
  3. The POM file (pom.xml)
  4. Payload.java
  5. PailStructurePayload.java
  6. PailPayload.java

  1. References
    See this page for more details about Pail:
    http://deals.manningpublications.com/pail.pdf
  2. The project Structure
    |+ mtitek-pail-object-samples
       |+ src
          |+ main
             |+ java
                |+ mtitek.pail.object.samples
                   |+ Payload.java
                   |+ PailStructurePayload.java
                   |+ PailPayload.java
       |+ pom.xml
  3. The POM file (pom.xml)
    <project xmlns="http://maven.apache.org/POM/4.0.0"
        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
        xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
        <modelVersion>4.0.0</modelVersion>
    
        <groupId>mtitek.pail.object.samples</groupId>
        <artifactId>mtitek-pail-object-samples</artifactId>
        <version>1.0.0-SNAPSHOT</version>
    
        <dependencies>
            <dependency>
                <groupId>org.apache.hadoop</groupId>
                <artifactId>hadoop-common</artifactId>
                <version>2.7.3</version>
            </dependency>
    
            <dependency>
                <groupId>com.backtype</groupId>
                <artifactId>dfs-datastores</artifactId>
                <version>1.3.6</version>
            </dependency>
        </dependencies>
    </project>
  4. Payload.java
    A simple POJO java class that will be used as an example to show how to write/read Objects to/from pail.

    ► src/main/java/mtitek/pail/object/samples/Payload.java

    package mtitek.pail.object.samples;
    
    public class Payload implements java.io.Serializable {
        private static final long serialVersionUID = -4058347141743162269L;
    
        private Integer field1;
    
        public Integer getField1() {
            return field1;
        }
    
        public void setField1(Integer field1) {
            this.field1 = field1;
        }
    
        @Override
        public String toString() {
            return "Payload [field1=" + field1 + "]";
        }
    }
    
  5. PailStructurePayload.java
    This java class is an implementation of the PailStructure interface that the Pail API will use to serialize/deserialize the payload objects.

    ► src/main/java/mtitek/pail/object/samples/PailStructurePayload.java

    package mtitek.pail.object.samples;
    
    import java.util.Collections;
    import java.util.List;
    
    import org.apache.commons.lang.SerializationUtils;
    
    import com.backtype.hadoop.pail.PailStructure;
    
    public class PailStructurePayload implements PailStructure<Payload> {
        private static final long serialVersionUID = 4759079091041385329L;
    
        public byte[] serialize(Payload object) {
            return SerializationUtils.serialize(object);
        }
    
        public Payload deserialize(byte[] serialized) {
            return (Payload) SerializationUtils.deserialize(serialized);
        }
    
        public Class<Payload> getType() {
            return Payload.class;
        }
    
        public boolean isValidTarget(String... dirs) {
            return true;
        }
    
        public List<String> getTarget(Payload object) {
            return Collections.emptyList();
        }
    }
  6. PailPayload.java
    This java class shows how to use the Pail API to write/read Objects to/from pail.

    ► src/main/java/mtitek/pail/object/samples/PailPayload.java

    package mtitek.pail.object.samples;
    
    import java.io.IOException;
    import java.util.Random;
    
    import org.apache.hadoop.fs.Path;
    
    import com.backtype.hadoop.pail.Pail;
    
    public class PailPayload {
        private static final String PAIL_PATH = "/tmp/pail-test/pail-payload";
    
        public static void main(String[] args) throws IOException {
            @SuppressWarnings("unchecked")
            final Pail<Payload> pail = Pail.create(PailString.PAIL_PATH, new PailStructurePayload(), false);
    
            writePayloads(pail);
    
            readPayload(pail);
        }
    
        private static void writePayloads(final Pail<Payload> pail) throws IOException {
            final Pail<Payload>.TypedRecordOutputStream typedRecordOutputStream = pail.openWrite();
    
            final Payload payload = new Payload();
    
            payload.setField1(new Random().nextInt());
    
            typedRecordOutputStream.writeObject(payload);
    
            typedRecordOutputStream.close();
        }
    
        private static void readPayload(final Pail<Payload> pail) throws IOException {
            for (final Payload payload : pail) {
                System.out.println(payload);
            }
        }
    }
© 2025  mtitek