[279014] [dstore][encoding] text file corruption can occur when downloading from UTF8 to cp1252

rse/plugins/org.eclipse.rse.services.dstore/src/org/eclipse/rse/internal/services/dstore/files/DStoreFileService.java

@@ -56,6 +56,7 @@
  * David McKnight   (IBM)        - [270468] [dstore] FileServiceSubSystem.list() returns folders when only FILE_TYPE_FILES is requested
  * David McKnight   (IBM)        - [272335] [dstore] not handling case where upload fails
  * David McKnight   (IBM)        - [278411] [dstore] upload status needs to be created in standard form when using windows server
+ * David McKnight   (IBM)        - [279014] [dstore][encoding] text file corruption can occur when downloading from UTF8 to cp1252
  *******************************************************************************/
 
 package org.eclipse.rse.internal.services.dstore.files;
@@ -87,6 +88,7 @@ import org.eclipse.dstore.core.model.IDataStoreProvider;
 import org.eclipse.osgi.util.NLS;
 import org.eclipse.rse.dstore.universal.miners.IUniversalDataStoreConstants;
 import org.eclipse.rse.dstore.universal.miners.UniversalByteStreamHandler;
+import org.eclipse.rse.internal.services.RSEServicesMessages;
 import org.eclipse.rse.internal.services.dstore.Activator;
 import org.eclipse.rse.internal.services.dstore.IDStoreMessageIds;
 import org.eclipse.rse.internal.services.dstore.ServiceResources;
@@ -822,7 +824,25 @@ public class DStoreFileService extends AbstractDStoreService implements IFileSer
 
 					IFileServiceCodePageConverter codePageConverter = CodePageConverterManager.getCodePageConverter(encoding, this);
 
-					codePageConverter.convertFileFromRemoteEncoding(remotePath, localFile, encoding, localEncoding, this);
+					try {
+						codePageConverter.convertFileFromRemoteEncoding(remotePath, localFile, encoding, localEncoding, this);
+					}
+					catch (RuntimeException e){
+						Throwable ex = e.getCause();
+						StringBuffer msgTxtBuffer = new StringBuffer(RSEServicesMessages.FILEMSG_OPERATION_FAILED);
+						msgTxtBuffer.append('\n');
+						msgTxtBuffer.append('\n');
+						msgTxtBuffer.append(remotePath);
+						msgTxtBuffer.append('\n');						
+						msgTxtBuffer.append(encoding);
+						msgTxtBuffer.append(" -> ");
+						msgTxtBuffer.append(localEncoding);
+						
+						SystemMessage msg = new SimpleSystemMessage(Activator.PLUGIN_ID,
+								IDStoreMessageIds.FILEMSG_IO_ERROR,
+								IStatus.ERROR, msgTxtBuffer.toString(), ex);
+						throw new SystemMessageException(msg);
+					}
 				}
 			}
 			else if (resultChild.getType().equals(IUniversalDataStoreConstants.DOWNLOAD_RESULT_FILE_NOT_FOUND_EXCEPTION))
@@ -1051,7 +1071,25 @@ public class DStoreFileService extends AbstractDStoreService implements IFileSer
 							String localEncoding = SystemEncodingUtil.getInstance().getLocalDefaultEncoding();
 							IFileServiceCodePageConverter codePageConverter = CodePageConverterManager.getCodePageConverter(hostEncodings[i], this);
 
-							codePageConverter.convertFileFromRemoteEncoding(remoteElement.getName(), localFile, hostEncodings[i], localEncoding, this);
+							try {
+								codePageConverter.convertFileFromRemoteEncoding(remoteElement.getName(), localFile, hostEncodings[i], localEncoding, this);
+							}
+							catch (RuntimeException e){
+								Throwable ex = e.getCause();
+								StringBuffer msgTxtBuffer = new StringBuffer(RSEServicesMessages.FILEMSG_OPERATION_FAILED);
+								msgTxtBuffer.append('\n');
+								msgTxtBuffer.append('\n');
+								msgTxtBuffer.append(remoteFiles[i]);
+								msgTxtBuffer.append('\n');						
+								msgTxtBuffer.append(hostEncodings[i]);
+								msgTxtBuffer.append(" -> ");
+								msgTxtBuffer.append(localEncoding);
+								
+								SystemMessage msg = new SimpleSystemMessage(Activator.PLUGIN_ID,
+										IDStoreMessageIds.FILEMSG_IO_ERROR,
+										IStatus.ERROR, msgTxtBuffer.toString(), ex);
+								throw new SystemMessageException(msg);
+							}
 						}
 					}
 					else if (resultChild.getType().equals(IUniversalDataStoreConstants.DOWNLOAD_RESULT_FILE_NOT_FOUND_EXCEPTION))

rse/plugins/org.eclipse.rse.services/src/org/eclipse/rse/services/files/DefaultFileServiceCodePageConverter.java

@@ -1,5 +1,5 @@
 /********************************************************************************
- * Copyright (c) 2007, 2008 IBM Corporation. All rights reserved.
+ * Copyright (c) 2007, 2009 IBM Corporation. All rights reserved.
  * This program and the accompanying materials are made available under the terms
  * of the Eclipse Public License v1.0 which accompanies this distribution, and is
  * available at http://www.eclipse.org/legal/epl-v10.html
@@ -12,6 +12,7 @@
  * David McKnight    (IBM)  -[209704] [api][dstore] Ability to override default encoding conversion needed.
  * David McKnight    (IBM)  -[220379] [api] Provide a means for contributing custom BIDI encodings
  * David McKnight    (IBM)  -[246857] Rename problem when a file is opened in the editor
+ * David McKnight    (IBM)  -[279014] [dstore][encoding] text file corruption can occur when downloading from UTF8 to cp1252
  ********************************************************************************/
 package org.eclipse.rse.services.files;
 
@@ -20,6 +21,11 @@ import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.CharBuffer;
+import java.nio.charset.Charset;
+import java.nio.charset.CharsetDecoder;
+import java.nio.charset.CharsetEncoder;
 
 /**
  * @since 3.0
@@ -51,30 +57,43 @@ public class DefaultFileServiceCodePageConverter implements
 				inputStream = new FileInputStream(file);
 				BufferedInputStream bufInputStream = new BufferedInputStream(inputStream, fileLength);
 				byte[] buffer = new byte[fileLength];
-				int bytesRead = bufInputStream.read(buffer, 0, fileLength);
+				bufInputStream.read(buffer, 0, fileLength);
 				bufInputStream.close();
-				inputStream.close();
+				ByteBuffer rmtBuf = ByteBuffer.wrap(buffer, 0, fileLength);
-	
+
-				byte[] localBuffer = new String(buffer, 0, bytesRead, remoteEncoding).getBytes(localEncoding);
+				// decoder to go from remote encoding to UTF8
-	
+				Charset rmtCharset = Charset.forName(remoteEncoding);
+				CharsetDecoder rmtDecoder = rmtCharset.newDecoder();
+
+				// convert from the remote encoding
+				CharBuffer decodedBuf = null;
+				decodedBuf = rmtDecoder.decode(rmtBuf);
+				// for conversion to the local encoding
+				Charset charset = Charset.forName(localEncoding);
+				CharsetEncoder encoder = charset.newEncoder();
+				byte[] localBuffer = null;
+				// convert to the specified local encoding
+				ByteBuffer lclBuf = encoder.encode(decodedBuf);
+				localBuffer = lclBuf.array();
 				outStream = new FileOutputStream(file);
 				outStream.write(localBuffer, 0, localBuffer.length);
-				outStream.close();
 			}
-		}
+		} catch (Exception e) {
-		catch (Exception e)
+			// outstream could not be written properly: report
-		{
+			throw new RuntimeException(e);
-			try {
+		} finally {
-				if (inputStream != null){
+			if (inputStream != null) {
+				try {
 					inputStream.close();
-				}	
+				} catch (IOException ioe) {
-				if (outStream != null){
-					outStream.close();
 				}
 			}
-			catch (IOException ioe){			
+			if (outStream != null) {
+				try {
+					outStream.close();
+				} catch (IOException ioe) {
+				}
 			}
-			
 		}
 	}
 

rse/plugins/org.eclipse.rse.services/src/org/eclipse/rse/services/files/IFileServiceCodePageConverter.java

@@ -1,33 +1,35 @@
 /********************************************************************************
- * Copyright (c) 2007, 2008 IBM Corporation. All rights reserved.
+ * Copyright (c) 2007, 2009 IBM Corporation. All rights reserved.
  * This program and the accompanying materials are made available under the terms
- * of the Eclipse Public License v1.0 which accompanies this distribution, and is 
+ * of the Eclipse Public License v1.0 which accompanies this distribution, and is
  * available at http://www.eclipse.org/legal/epl-v10.html
- * 
+ *
  * Initial Contributors:
  * The following IBM employees contributed to the Remote System Explorer
  * component that contains this file: David McKnight.
- * 
+ *
  * Contributors:
  * David McKnight    (IBM)  -[209704] [api] Ability to override default encoding conversion needed.
  * David McKnight    (IBM)  -[220379] [api] Provide a means for contributing custom BIDI encodings
+ * David McKnight    (IBM)  -[279014] [dstore][encoding] text file corruption can occur when downloading from UTF8 to cp1252
  ********************************************************************************/
 
 package org.eclipse.rse.services.files;
 
 import java.io.File;
+import java.nio.charset.CharacterCodingException;
 
 
 /**
- * This interface is used by the extension point 
+ * This interface is used by the extension point
- * It allows overriding the Universal File Subsystem translation of files, and results in 
+ * It allows overriding the Universal File Subsystem translation of files, and results in
  * binary transfer, with calls to the implementor to handle code page conversion.
  * @since org.eclipse.rse.services 3.0
  */
 public interface IFileServiceCodePageConverter {
 
 	/**
-	 * Converts a client string to remote bytes, for use when uploading in binary mode.  
+	 * Converts a client string to remote bytes, for use when uploading in binary mode.
 	 * @param remotePath        the path of the remote file
 	 * @param clientString		the client string to convert
 	 * @param remoteEncoding	The remote encoding for the desired server bytes
@@ -36,17 +38,18 @@ public interface IFileServiceCodePageConverter {
 	 * @return					The bytes to upload to the server
 	 */
 	public byte [] convertClientStringToRemoteBytes(String remotePath, String clientString, String remoteEncoding, IFileService fs);
-	
+
 	/**
 	 * Converts the specified file (which was downloaded from the server in binary mode) from server encoding bytes, to local encoding
 	 * @param remotePath        the path of the remote file
-	 * @param file				The file to convert 
+	 * @param file				The file to convert
 	 * @param localEncoding		The remote encoding of the file
 	 * @param fs                The file service to apply conversion to.
 	 *                          Can be used to determine implementation specific settings to the converter
+	 *  @throws RuntimeException (wrapping a CharacterCodingException or IOException) in case of an error transposing from source to target encoding
 	 */
 	public void convertFileFromRemoteEncoding(String remotePath, File file, String remoteEncoding, String localEncoding, IFileService fs);
-	
+
 	/**
 	 * Indicates whether or not the specified server encoding and subsystem implementation is supported by this code page converter
 	 * @param remoteEncoding		The remote encoding from the server to check
@@ -55,12 +58,12 @@ public interface IFileServiceCodePageConverter {
 	 * @return						True if this code page converter can convert the specified encoding, false otherwise
 	 */
 	public boolean isServerEncodingSupported(String remoteEncoding, IFileService fs);
-	
+
 	/**
 	 * Indicates the priority of this code page converter if more than one code page converter
 	 * handle a particular encoding.  The lower the number, the higher the priority.
 	 * @return priority
 	 */
 	public int getPriority(String remoteEncoding, IFileService fs);
-		
+
 }