client.java(org\apache\nutch\protocol\ftp)
client.java为Nutch提供这样的功能：获取Ftp服务器上的文件列表，并能提供能从服务器上下传文件的功能。这个类Client(继承自ftp)考虑到了与服务器交互的所有的底层细节，并且提供了方便的高层接口。
这个类是修改自apache commons-net的FtpClient.java
(1) Use stream mode for data tranfer. Block mode will be better for
* multiple file downloading and partial file downloading. However
* not every ftpd has block mode support.
* (2) Use passive mode for data connection.
* So nutch will work if we run behind firewall.
* (3) Data connection is opened/closed per ftp command for the reasons
* listed in (1). There are ftp servers out there,
* when partial downloading is enforeced by closing data channel
* socket on our client side, the server side immediately closes
* control channel (socket). Our codes deal with such a bad behavior.
* (4) LIST is used to obtain remote file attributes if possible.
* MDTM & SIZE would be nice, but not as ubiquitously implemented as LIST.
* (5) Avoid using ABOR in single thread? Do not use it at all.
以下是开放的公用方法
/***
* Enable or disable verification that the remote host taking part
* of a data connection is the same as the host to which the control
* connection is attached.The default is for verification to be
* enabled.You may set this value at any time, whether the
* FTPClient is currently connected or not.
*
* @param enable True to enable verification, false to disable verification.
***/
public void setRemoteVerificationEnabled(boolean enable)
{
__remoteVerificationEnabled = enable;
}
/***
* Return whether or not verification of the remote host participating
* in data connections is enabled.The default behavior is for
* verification to be enabled.
*
* @return True if verification is enabled, false if not.
***/
public boolean isRemoteVerificationEnabled()
{
return __remoteVerificationEnabled;
}
/***
* Login to the FTP server using the provided username and password.
*
* @param username The username to login under.
* @param password The password to use.
* @return True if successfully completed, false if not.
* @exception FTPConnectionClosedException
* If the FTP server prematurely closes the connection as a result
* of the client being idle or some other reason causing the server
* to send FTP reply code 421.This exception may be caught either
* as an IOException or independently as itself.
* @exception IOExceptionIf an I/O error occurs while either sending a
* command to the server or receiving a reply from the server.
***/
public boolean login(String username, String password) throws IOException
{
user(username);
if (FTPReply.isPositiveCompletion(getReplyCode()))
return true;
// If we get here, we either have an error code, or an intermmediate
// reply requesting password.
if (!FTPReply.isPositiveIntermediate(getReplyCode()))
return false;
return FTPReply.isPositiveCompletion(pass(password));
}
/***
* Logout of the FTP server by sending the QUIT command.
*
* @return True if successfully completed, false if not.
* @exception FTPConnectionClosedException
* If the FTP server prematurely closes the connection as a result
* of the client being idle or some other reason causing the server
* to send FTP reply code 421.This exception may be caught either
* as an IOException or independently as itself.
* @exception IOExceptionIf an I/O error occurs while either sending a
* command to the server or receiving a reply from the server.
***/
public boolean logout() throws IOException
{
return FTPReply.isPositiveCompletion(quit());
}
// retrieve list reply for path
public void retrieveList(String path, List entries, int limit,
FTPFileEntryParser parser)
throws IOException,
FtpExceptionCanNotHaveDataConnection,
FtpExceptionUnknownForcedDataClose,
FtpExceptionControlClosedByForcedDataClose {
Socket socket = __openPassiveDataConnection(FTPCommand.LIST, path);
if (socket == null)
throw new FtpExceptionCanNotHaveDataConnection("LIST "
+ ((path == null) ? "" : path));
BufferedReader reader =
new BufferedReader(new InputStreamReader(socket.getInputStream()));
// force-close data channel socket, when download limit is reached
boolean mandatory_close = false;
//List entries = new LinkedList();
int count = 0;
String line = parser.readNextEntry(reader);
while (line != null) {
FTPFile ftpFile = parser.parseFTPEntry(line);
// skip non-formatted lines
if (ftpFile == null) {
line = parser.readNextEntry(reader);
continue;
}
entries.add(ftpFile);
count += line.length();
// impose download limit if limit >= 0, otherwise no limit
// here, cut off is up to the line when total bytes is just over limit
if (limit >= 0 && count > limit) {
mandatory_close = true;
break;
}
line = parser.readNextEntry(reader);
}
//if (mandatory_close)
// you always close here, no matter mandatory_close or not.
// however different ftp servers respond differently, see below.
socket.close();
// scenarios:
// (1) mandatory_close is false, download limit not reached
// no special care here
// (2) mandatory_close is true, download limit is reached
// different servers have different reply codes:
try {
int reply = getReply();
if (!_notBadReply(reply))
throw new FtpExceptionUnknownForcedDataClose(getReplyString());
} catch (FTPConnectionClosedException e) {
// some ftp servers will close control channel if data channel socket
// is closed by our end before all data has been read out. Check:
// tux414.q-tam.hp.com FTP server (hp.com version whp02)
// so must catch FTPConnectionClosedException thrown by getReply() above
//disconnect();
throw new FtpExceptionControlClosedByForcedDataClose(e.getMessage());
}
}
// retrieve file for path
public void retrieveFile(String path, OutputStream os, int limit)
throws IOException,
FtpExceptionCanNotHaveDataConnection,
FtpExceptionUnknownForcedDataClose,
FtpExceptionControlClosedByForcedDataClose {
Socket socket = __openPassiveDataConnection(FTPCommand.RETR, path);
if (socket == null)
throw new FtpExceptionCanNotHaveDataConnection("RETR "
+ ((path == null) ? "" : path));
InputStream input = socket.getInputStream();
// 20040318, xing, treat everything as BINARY_FILE_TYPE for now
// do we ever need ASCII_FILE_TYPE?
//if (__fileType == ASCII_FILE_TYPE)
// input = new FromNetASCIIInputStream(input);
// fixme, should we instruct server here for binary file type?
// force-close data channel socket
boolean mandatory_close = false;
int len; int count = 0;
byte[] buf =
new byte;
while((len=input.read(buf,0,buf.length)) != -1){
count += len;
// impose download limit if limit >= 0, otherwise no limit
// here, cut off is exactly of limit bytes
if (limit >= 0 && count > limit) {
os.write(buf,0,len-(count-limit));
mandatory_close = true;
break;
}
os.write(buf,0,len);
os.flush();
}
//if (mandatory_close)
// you always close here, no matter mandatory_close or not.
// however different ftp servers respond differently, see below.
socket.close();
// scenarios:
// (1) mandatory_close is false, download limit not reached
// no special care here
// (2) mandatory_close is true, download limit is reached
// different servers have different reply codes:
// do not need this
//sendCommand("ABOR");
try {
int reply = getReply();
if (!_notBadReply(reply))
throw new FtpExceptionUnknownForcedDataClose(getReplyString());
} catch (FTPConnectionClosedException e) {
// some ftp servers will close control channel if data channel socket
// is closed by our end before all data has been read out. Check:
// tux414.q-tam.hp.com FTP server (hp.com version whp02)
// so must catch FTPConnectionClosedException thrown by getReply() above
//disconnect();
throw new FtpExceptionControlClosedByForcedDataClose(e.getMessage());
}
}
// reply check after closing data connection
private boolean _notBadReply(int reply) {
if (FTPReply.isPositiveCompletion(reply)) {
// do nothing
} else if (reply == 426) { // FTPReply.TRANSFER_ABORTED
// some ftp servers reply 426, e.g.,
// foggy FTP server (Version wu-2.6.2(2)
// there is second reply witing? no!
//getReply();
} else if (reply == 450) { // FTPReply.FILE_ACTION_NOT_TAKEN
// some ftp servers reply 450, e.g.,
// ProFTPD
// there is second reply witing? no!
//getReply();
} else if (reply == 451) { // FTPReply.ACTION_ABORTED
// some ftp servers reply 451, e.g.,
// ProFTPD
// there is second reply witing? no!
//getReply();
} else if (reply == 451) { // FTPReply.ACTION_ABORTED
} else {
// what other kind of ftp server out there?
return false;
}
return true;
}
/***
* Sets the file type to be transferred.This should be one of
*FTP.ASCII_FILE_TYPE ,FTP.IMAGE_FILE_TYPE ,
* etc.The file type only needs to be set when you want to change the
* type.After changing it, the new type stays in effect until you change
* it again.The default file type isFTP.ASCII_FILE_TYPE
* if this method is never called.
*
* @param fileType The_FILE_TYPEconstant indcating the
* type of file.
* @return True if successfully completed, false if not.
* @exception FTPConnectionClosedException
* If the FTP server prematurely closes the connection as a result
* of the client being idle or some other reason causing the server
* to send FTP reply code 421.This exception may be caught either
* as an IOException or independently as itself.
* @exception IOExceptionIf an I/O error occurs while either sending a
* command to the server or receiving a reply from the server.
***/
public boolean setFileType(int fileType) throws IOException
{
if (FTPReply.isPositiveCompletion(type(fileType)))
{
__fileType = fileType;
__fileFormat = FTP.NON_PRINT_TEXT_FORMAT;
return true;
}
return false;
}
/***
* Fetches the system type name from the server and returns the string.
* This value is cached for the duration of the connection after the
* first call to this method.In other words, only the first time
* that you invoke this method will it issue a SYST command to the
* FTP server.FTPClient will remember the value and return the
* cached value until a call to disconnect.
*
* @return The system type name obtained from the server.null if the
* information could not be obtained.
* @exception FTPConnectionClosedException
* If the FTP server prematurely closes the connection as a result
* of the client being idle or some other reason causing the server
* to send FTP reply code 421.This exception may be caught either
* as an IOException or independently as itself.
* @exception IOExceptionIf an I/O error occurs while either sending a
*command to the server or receiving a reply from the server.
***/
public String getSystemName()
throws IOException, FtpExceptionBadSystResponse
{
//if (syst() == FTPReply.NAME_SYSTEM_TYPE)
// Technically, we should expect a NAME_SYSTEM_TYPE response, but
// in practice FTP servers deviate, so we soften the condition to
// a positive completion.
if (__systemName == null && FTPReply.isPositiveCompletion(syst())) {
__systemName = (getReplyStrings()).substring(4);
} else {
throw new FtpExceptionBadSystResponse(
"Bad response of SYST: " + getReplyString());
}
return __systemName;
}
/***
* Sends a NOOP command to the FTP server.This is useful for preventing
* server timeouts.
*
* @return True if successfully completed, false if not.
* @exception FTPConnectionClosedException
* If the FTP server prematurely closes the connection as a result
* of the client being idle or some other reason causing the server
* to send FTP reply code 421.This exception may be caught either
* as an IOException or independently as itself.
* @exception IOExceptionIf an I/O error occurs while either sending a
* command to the server or receiving a reply from the server.
***/
public boolean sendNoOp() throws IOException
{
return FTPReply.isPositiveCompletion(noop());
}
页:
[1]