HDFS中的DataNode菊花链式接收数据
HDFS中的DataNode菊花链式接收数据
强子哥哥 发表于3年前
HDFS中的DataNode菊花链式接收数据
  • 发表于 3年前
  • 阅读 42
  • 收藏 0
  • 点赞 0
  • 评论 0

标题:腾讯云 新注册用户域名抢购1元起>>>   

public void run() {//数据节点运行

            try {

                DataInputStream in = new DataInputStream(new BufferedInputStream(s.getInputStream()));

                try {//获取输入流

                    byte op = (byte) in.read();//读取操作类型

                    if (op == OP_WRITE_BLOCK) {//如果是写一个块

                        /

                        // Read in the header

                        //

                        DataOutputStream reply = new DataOutputStream(new BufferedOutputStream(s.getOutputStream()));

                        try {//获取输出流

                            boolean shouldReportBlock = in.readBoolean();//读取一个boolean

                            Block b = new Block();

                            b.readFields(in);//创建一个新的block

                            int numTargets = in.readInt();//读取目标机器的个数

                            if (numTargets <= 0) {

                                throw new IOException("Mislabelled incoming datastream.");

                            }

                            DatanodeInfo targets[] = new DatanodeInfo[numTargets];//用来存储目标datanode节点信息

                            for (int i = 0; i < targets.length; i++) {//开始初始化it.

                                DatanodeInfo tmp = new DatanodeInfo();

                                tmp.readFields(in);

                                targets[i] = tmp;

                            }

                            byte encodingType = (byte) in.read();//获取encodingType

                            long len = in.readLong();//获取这个块的长度

                            //

                            // Make sure curTarget is equal to this machine

                            //

                            DatanodeInfo curTarget = targets[0];//获取第一个data节点

                            //

                            // Track all the places we've successfully written the block

                            //

                            Vector mirrors = new Vector();//???

                            //

                            // Open local disk out

                            //

                            DataOutputStream out = new DataOutputStream(new BufferedOutputStream(data.writeToBlock(b)));

//本地文件输出流

                            InetSocketAddress mirrorTarget = null;

                            try {

                                //

                                // Open network conn to backup machine, if 

                                // appropriate

                                //

                                DataInputStream in2 = null;

                                DataOutputStream out2 = null;

                                if (targets.length > 1) {

                                    // Connect to backup machine

                                    mirrorTarget = createSocketAddr(targets[1].getName().toString());//连接镜像节点

                                    try {

                                        Socket s2 = new Socket();

                                        s2.connect(mirrorTarget, READ_TIMEOUT);//连接镜像节点

                                        s2.setSoTimeout(READ_TIMEOUT);//设置超时时间

                                        out2 = new DataOutputStream(new BufferedOutputStream(s2.getOutputStream()));

                                        in2 = new DataInputStream(new BufferedInputStream(s2.getInputStream()));

                                        //获取与备份节点的2IO

                                        // Write connection header

                                        out2.write(OP_WRITE_BLOCK);//写操作类型

                                        out2.writeBoolean(shouldReportBlock);//boolean

                                        b.write(out2);//block

                                        out2.writeInt(targets.length - 1);//写剩下的几个datanode信息

                                        for (int i = 1; i < targets.length; i++) {

                                            targets[i].write(out2);

                                        }

                                        out2.write(encodingType);

                                        out2.writeLong(len);//数据长度

                                    } catch (IOException ie) {

                                        if (out2 != null) {

                                            try {

                                                out2.close();

                                                in2.close();

                                            } catch (IOException out2close) {

                                            } finally {

                                                out2 = null;

                                                in2 = null;

                                            }

                                        }

                                    }

                                }

                                //

                                // Process incoming data, copy to disk and

                                // maybe to network.

                                //万事俱备只欠数据

                                try {

                                    boolean anotherChunk = len != 0;//true

                                    byte buf[] = new byte[BUFFER_SIZE];//设置缓冲区

                                    while (anotherChunk) {

                                        while (len > 0) {//读取一些数据

                                            int bytesRead = in.read(buf, 0, (int)Math.min(buf.length, len));

                                            if (bytesRead < 0) {

                                              throw new EOFException("EOF reading from "+s.toString());

                                            }

                                            if (bytesRead > 0) {

                                                try {//写入到本地磁盘文件

                                                    out.write(buf, 0, bytesRead);

                                                } catch (IOException iex) {

                                                    shutdown();

                                                    throw iex;

                                                }

                                                if (out2 != null) {

                                                    try {//写入到网络以便让备份节点获取

                                                        out2.write(buf, 0, bytesRead);

                                                    } catch (IOException out2e) {

                                                        //

                                                        // If stream-copy fails, continue 

                                                        // writing to disk.  We shouldn't 

                                                        // interrupt client write.

                                                        //

                                                        try {

                                                            out2.close();

                                                            in2.close();

                                                        } catch (IOException out2close) {

                                                        } finally {

                                                            out2 = null;

                                                            in2 = null;

                                                        }

                                                    }

                                                }

                                                len -= bytesRead;//调整要写的字节数

                                            }

                                        }

                                        if (encodingType == RUNLENGTH_ENCODING) {

                                            anotherChunk = false;

                                        } else if (encodingType == CHUNKED_ENCODING) {

                                            len = in.readLong();//准备读取下一块block

                                            if (out2 != null) {

                                                out2.writeLong(len);

                                            }

                                            if (len == 0) {

                                                anotherChunk = false;

                                            }

                                        }

                                    }

                                    if (out2 == null) {

                                        LOG.info("Received block " + b + " from " + s.getInetAddress());

                                    } else {

                                        out2.flush();//输出缓冲区的内容

                                        long complete = in2.readLong();//得到备份节点的响应

                                        if (complete != WRITE_COMPLETE) {

                                            LOG.info("Conflicting value for WRITE_COMPLETE: " + complete);

                                        }

                                        LocatedBlock newLB = new LocatedBlock();

                                        newLB.readFields(in2);//得到block以及成功写到的节点信息

                                        DatanodeInfo mirrorsSoFar[] = newLB.getLocations();

                                        for (int k = 0; k < mirrorsSoFar.length; k++) {

                                            mirrors.add(mirrorsSoFar[k]);//更新到本地的mirrors

                                        }

                                        LOG.info("Received block " + b + " from " + s.getInetAddress() + " and mirrored to " + mirrorTarget);

                                    }

                                } finally {

                                    if (out2 != null) {//清理线程

                                        out2.close();

                                        in2.close();

                                    }

                                }

                            } finally {//清理现场

                                try {

                                    out.close();

                                } catch (IOException iex) {

                                    shutdown();

                                    throw iex;

                                }

                            }

                            data.finalizeBlock(b);//写入到dirTree结构中

                            // 

                            // Tell the namenode that we've received this block 

                            // in full, if we've been asked to.  This is done

                            // during NameNode-directed block transfers, but not

                            // client writes.

                            //

                            if (shouldReportBlock) {

                                synchronized (receivedBlockList) {

                                    receivedBlockList.add(b);//更新本地的receivedBlockList

                                    receivedBlockList.notifyAll();

                                }

                            }

                            //

                            // Tell client job is done, and reply with

                            // the new LocatedBlock.

                            //

                            reply.writeLong(WRITE_COMPLETE);//写结束

                            mirrors.add(curTarget);//把自己加上来

                            LocatedBlock newLB = new LocatedBlock(b, (DatanodeInfo[]) mirrors.toArray(new DatanodeInfo[mirrors.size()]));

                            newLB.write(reply);//返回给客户端

                        } finally {

                            reply.close();

                        }

                        //剩下的为读数据,忽略。

                    } else if (op == OP_READ_BLOCK || op == OP_READSKIP_BLOCK) {

                        //

                        // Read in the header

                        //

                        Block b = new Block();

                        b.readFields(in);

                        long toSkip = 0;

                        if (op == OP_READSKIP_BLOCK) {

                            toSkip = in.readLong();

                        }

                        //

                        // Open reply stream

                        //

                        DataOutputStream out = new DataOutputStream(new BufferedOutputStream(s.getOutputStream()));

                        try {

                            //

                            // Write filelen of -1 if error

                            //

                            if (! data.isValidBlock(b)) {

                                out.writeLong(-1);

                            } else {

                                //

                                // Get blockdata from disk

                                //

                                long len = data.getLength(b);

                                DataInputStream in2 = new DataInputStream(data.getBlockData(b));

                                out.writeLong(len);

                                if (op == OP_READSKIP_BLOCK) {

                                    if (toSkip > len) {

                                        toSkip = len;

                                    }

                                    long amtSkipped = 0;

                                    try {

                                        amtSkipped = in2.skip(toSkip);

                                    } catch (IOException iex) {

                                        shutdown();

                                        throw iex;

                                    }

                                    out.writeLong(amtSkipped);

                                }

                                byte buf[] = new byte[BUFFER_SIZE];

                                try {

                                    int bytesRead = 0;

                                    try {

                                        bytesRead = in2.read(buf);

                                    } catch (IOException iex) {

                                        shutdown();

                                        throw iex;

                                    }

                                    while (bytesRead >= 0) {

                                        out.write(buf, 0, bytesRead);

                                        len -= bytesRead;

                                        try {

                                            bytesRead = in2.read(buf);

                                        } catch (IOException iex) {

                                            shutdown();

                                            throw iex;

                                        }

                                    }

                                } catch (SocketException se) {

                                    // This might be because the reader

                                    // closed the stream early

                                } finally {

                                    try {

                                        in2.close();

                                    } catch (IOException iex) {

                                        shutdown();

                                        throw iex;

                                    }

                                }

                            }

                            LOG.info("Served block " + b + " to " + s.getInetAddress());

                        } finally {

                            out.close();

                        }

                    } else {

                        while (op >= 0) {

                            System.out.println("Faulty op: " + op);

                            op = (byte) in.read();

                        }

                        throw new IOException("Unknown opcode for incoming data stream");

                    }

                } finally {

                    in.close();

                }

            } catch (IOException ie) {

              LOG.log(Level.WARNING"DataXCeiver", ie);

            } finally {

                try {

                    s.close();

                } catch (IOException ie2) {

                }

            }

        }

    }

标签: HDFS DataNode
共有 人打赏支持
强子哥哥
粉丝 839
博文 673
码字总数 711726
作品 8
×
强子哥哥
如果觉得我的文章对您有用,请随意打赏。您的支持将鼓励我继续创作!
* 金额(元)
¥1 ¥5 ¥10 ¥20 其他金额
打赏人
留言
* 支付类型
微信扫码支付
打赏金额:
已支付成功
打赏金额: